Add a "skip_actions" option to skip running particular actions (#701).

This commit is contained in:
Dan Helfman 2023-10-31 21:54:41 -07:00
parent c3efe1b90e
commit ef448e2dd1
10 changed files with 190 additions and 37 deletions

5
NEWS
View file

@ -1,4 +1,9 @@
1.8.5.dev0 1.8.5.dev0
* #701: Add a "skip_actions" option to skip running particular actions, handy for append-only or
checkless configurations. See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/set-up-backups/#skipping-actions
* #701: Deprecate the "disabled" value for the "checks" option in favor of the new "skip_actions"
option.
* #779: Add a "--match-archives" flag to the "check" action for selecting the archives to check, * #779: Add a "--match-archives" flag to the "check" action for selecting the archives to check,
overriding the existing "archive_name_format" and "match_archives" options in configuration. overriding the existing "archive_name_format" and "match_archives" options in configuration.
* #779: Only parse "--override" values as complex data types when they're for options of those * #779: Only parse "--override" values as complex data types when they're for options of those

View file

@ -39,7 +39,11 @@ def parse_checks(config, only_checks=None):
check_config['name'] for check_config in (config.get('checks', None) or DEFAULT_CHECKS) check_config['name'] for check_config in (config.get('checks', None) or DEFAULT_CHECKS)
) )
checks = tuple(check.lower() for check in checks) checks = tuple(check.lower() for check in checks)
if 'disabled' in checks: if 'disabled' in checks:
logger.warning(
'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead'
)
if len(checks) > 1: if len(checks) > 1:
logger.warning( logger.warning(
'Multiple checks are configured, but one of them is "disabled"; not running any checks' 'Multiple checks are configured, but one of them is "disabled"; not running any checks'
@ -119,6 +123,9 @@ def filter_checks_on_frequency(
Raise ValueError if a frequency cannot be parsed. Raise ValueError if a frequency cannot be parsed.
''' '''
if not checks:
return checks
filtered_checks = list(checks) filtered_checks = list(checks)
if force: if force:

View file

@ -70,6 +70,12 @@ def run_configuration(config_filename, config, arguments):
using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments)
monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity)
monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED
skip_actions = config.get('skip_actions')
if skip_actions:
logger.debug(
f"{config_filename}: Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions"
)
try: try:
local_borg_version = borg_version.local_borg_version(config, local_path) local_borg_version = borg_version.local_borg_version(config, local_path)
@ -274,6 +280,7 @@ def run_actions(
'repositories': ','.join([repo['path'] for repo in config['repositories']]), 'repositories': ','.join([repo['path'] for repo in config['repositories']]),
'log_file': global_arguments.log_file if global_arguments.log_file else '', 'log_file': global_arguments.log_file if global_arguments.log_file else '',
} }
skip_actions = set(config.get('skip_actions', {}))
command.execute_hook( command.execute_hook(
config.get('before_actions'), config.get('before_actions'),
@ -285,7 +292,7 @@ def run_actions(
) )
for action_name, action_arguments in arguments.items(): for action_name, action_arguments in arguments.items():
if action_name == 'rcreate': if action_name == 'rcreate' and action_name not in skip_actions:
borgmatic.actions.rcreate.run_rcreate( borgmatic.actions.rcreate.run_rcreate(
repository, repository,
config, config,
@ -295,7 +302,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'transfer': elif action_name == 'transfer' and action_name not in skip_actions:
borgmatic.actions.transfer.run_transfer( borgmatic.actions.transfer.run_transfer(
repository, repository,
config, config,
@ -305,7 +312,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'create': elif action_name == 'create' and action_name not in skip_actions:
yield from borgmatic.actions.create.run_create( yield from borgmatic.actions.create.run_create(
config_filename, config_filename,
repository, repository,
@ -318,7 +325,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'prune': elif action_name == 'prune' and action_name not in skip_actions:
borgmatic.actions.prune.run_prune( borgmatic.actions.prune.run_prune(
config_filename, config_filename,
repository, repository,
@ -331,7 +338,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'compact': elif action_name == 'compact' and action_name not in skip_actions:
borgmatic.actions.compact.run_compact( borgmatic.actions.compact.run_compact(
config_filename, config_filename,
repository, repository,
@ -344,7 +351,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'check': elif action_name == 'check' and action_name not in skip_actions:
if checks.repository_enabled_for_checks(repository, config): if checks.repository_enabled_for_checks(repository, config):
borgmatic.actions.check.run_check( borgmatic.actions.check.run_check(
config_filename, config_filename,
@ -357,7 +364,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'extract': elif action_name == 'extract' and action_name not in skip_actions:
borgmatic.actions.extract.run_extract( borgmatic.actions.extract.run_extract(
config_filename, config_filename,
repository, repository,
@ -369,7 +376,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'export-tar': elif action_name == 'export-tar' and action_name not in skip_actions:
borgmatic.actions.export_tar.run_export_tar( borgmatic.actions.export_tar.run_export_tar(
repository, repository,
config, config,
@ -379,7 +386,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'mount': elif action_name == 'mount' and action_name not in skip_actions:
borgmatic.actions.mount.run_mount( borgmatic.actions.mount.run_mount(
repository, repository,
config, config,
@ -389,7 +396,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'restore': elif action_name == 'restore' and action_name not in skip_actions:
borgmatic.actions.restore.run_restore( borgmatic.actions.restore.run_restore(
repository, repository,
config, config,
@ -399,7 +406,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'rlist': elif action_name == 'rlist' and action_name not in skip_actions:
yield from borgmatic.actions.rlist.run_rlist( yield from borgmatic.actions.rlist.run_rlist(
repository, repository,
config, config,
@ -409,7 +416,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'list': elif action_name == 'list' and action_name not in skip_actions:
yield from borgmatic.actions.list.run_list( yield from borgmatic.actions.list.run_list(
repository, repository,
config, config,
@ -419,7 +426,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'rinfo': elif action_name == 'rinfo' and action_name not in skip_actions:
yield from borgmatic.actions.rinfo.run_rinfo( yield from borgmatic.actions.rinfo.run_rinfo(
repository, repository,
config, config,
@ -429,7 +436,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'info': elif action_name == 'info' and action_name not in skip_actions:
yield from borgmatic.actions.info.run_info( yield from borgmatic.actions.info.run_info(
repository, repository,
config, config,
@ -439,7 +446,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'break-lock': elif action_name == 'break-lock' and action_name not in skip_actions:
borgmatic.actions.break_lock.run_break_lock( borgmatic.actions.break_lock.run_break_lock(
repository, repository,
config, config,
@ -449,7 +456,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'export': elif action_name == 'export' and action_name not in skip_actions:
borgmatic.actions.export_key.run_export_key( borgmatic.actions.export_key.run_export_key(
repository, repository,
config, config,
@ -459,7 +466,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
) )
elif action_name == 'borg': elif action_name == 'borg' and action_name not in skip_actions:
borgmatic.actions.borg.run_borg( borgmatic.actions.borg.run_borg(
repository, repository,
config, config,

View file

@ -1,9 +1,9 @@
def repository_enabled_for_checks(repository, consistency): def repository_enabled_for_checks(repository, config):
''' '''
Given a repository name and a consistency configuration dict, return whether the repository Given a repository name and a configuration dict, return whether the
is enabled to have consistency checks run. repository is enabled to have consistency checks run.
''' '''
if not consistency.get('check_repositories'): if not config.get('check_repositories'):
return True return True
return repository in consistency['check_repositories'] return repository in config['check_repositories']

View file

@ -423,7 +423,9 @@ properties:
command-line invocation. command-line invocation.
keep_within: keep_within:
type: string type: string
description: Keep all archives within this time interval. description: |
Keep all archives within this time interval. See "skip_actions" for
disabling pruning altogether.
example: 3H example: 3H
keep_secondly: keep_secondly:
type: integer type: integer
@ -479,13 +481,13 @@ properties:
- disabled - disabled
description: | description: |
Name of consistency check to run: "repository", Name of consistency check to run: "repository",
"archives", "data", and/or "extract". Set to "disabled" "archives", "data", and/or "extract". "repository"
to disable all consistency checks. "repository" checks checks the consistency of the repository, "archives"
the consistency of the repository, "archives" checks all checks all of the archives, "data" verifies the
of the archives, "data" verifies the integrity of the integrity of the data within the archives, and "extract"
data within the archives, and "extract" does an does an extraction dry-run of the most recent archive.
extraction dry-run of the most recent archive. Note that Note that "data" implies "archives". See "skip_actions"
"data" implies "archives". for disabling checks altogether.
example: repository example: repository
frequency: frequency:
type: string type: string
@ -525,6 +527,18 @@ properties:
Apply color to console output. Can be overridden with --no-color Apply color to console output. Can be overridden with --no-color
command-line flag. Defaults to true. command-line flag. Defaults to true.
example: false example: false
skip_actions:
type: array
items:
type: string
description: |
List of one or more actions to skip running for this configuration
file, even if specified on the command-line (explicitly or
implicitly). This is handy for append-only configurations where you
never want to run "compact" or checkless configuration where you
want to skip "check". Defaults to not skipping any actions.
example:
- compact
before_actions: before_actions:
type: array type: array
items: items:

View file

@ -162,7 +162,16 @@ location:
If that's still too slow, you can disable consistency checks entirely, If that's still too slow, you can disable consistency checks entirely,
either for a single repository or for all repositories. either for a single repository or for all repositories.
Disabling all consistency checks looks like this: <span class="minilink minilink-addedin">New in version 1.8.5</span> Disabling
all consistency checks looks like this:
```yaml
skip_actions:
- check
```
<span class="minilink minilink-addedin">Prior to version 1.8.5</span> Use this
configuration instead:
```yaml ```yaml
checks: checks:
@ -170,10 +179,10 @@ checks:
``` ```
<span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put <span class="minilink minilink-addedin">Prior to version 1.8.0</span> Put
this option in the `consistency:` section of your configuration. `checks:` in the `consistency:` section of your configuration.
<span class="minilink minilink-addedin">Prior to version 1.6.2</span> `checks` <span class="minilink minilink-addedin">Prior to version 1.6.2</span>
was a plain list of strings without the `name:` part. For instance: `checks:` was a plain list of strings without the `name:` part. For instance:
```yaml ```yaml
checks: checks:

View file

@ -282,6 +282,21 @@ due to things like file damage. For instance:
sudo borgmatic --verbosity 1 --list --stats sudo borgmatic --verbosity 1 --list --stats
``` ```
### Skipping actions
<span class="minilink minilink-addedin">New in version 1.8.5</span> You can
configure borgmatic to skip running certain actions (default or otherwise).
For instance, to always skip the `compact` action when using [Borg's
append-only
mode](https://borgbackup.readthedocs.io/en/stable/usage/notes.html#append-only-mode-forbid-compaction),
set the `skip_actions` option:
```
skip_actions:
- compact
```
## Autopilot ## Autopilot
Running backups manually is good for validating your configuration, but I'm Running backups manually is good for validating your configuration, but I'm

View file

@ -193,6 +193,19 @@ def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_
) == ('archives',) ) == ('archives',)
def test_filter_checks_on_frequency_passes_through_empty_checks():
assert (
module.filter_checks_on_frequency(
config={'checks': [{'name': 'archives', 'frequency': '1 hour'}]},
borg_repository_id='repo',
checks=(),
force=False,
archives_check_id='1234',
)
== ()
)
def test_make_archive_filter_flags_with_default_checks_and_prefix_returns_default_flags(): def test_make_archive_filter_flags_with_default_checks_and_prefix_returns_default_flags():
flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.feature).should_receive('available').and_return(True)
flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(())

View file

@ -23,6 +23,16 @@ def test_run_configuration_runs_actions_for_each_repository():
assert results == expected_results assert results == expected_results
def test_run_configuration_with_skip_actions_does_not_raise():
flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO)
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module).should_receive('run_actions').and_return(flexmock()).and_return(flexmock())
config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'skip_actions': ['compact']}
arguments = {'global': flexmock(monitoring_verbosity=1)}
list(module.run_configuration('test.yaml', config, arguments))
def test_run_configuration_with_invalid_borg_version_errors(): def test_run_configuration_with_invalid_borg_version_errors():
flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO)
flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError) flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError)
@ -504,6 +514,24 @@ def test_run_actions_runs_create():
assert result == (expected,) assert result == (expected,)
def test_run_actions_with_skip_actions_skips_create():
flexmock(module).should_receive('add_custom_log_levels')
flexmock(module.command).should_receive('execute_hook')
flexmock(borgmatic.actions.create).should_receive('run_create').never()
tuple(
module.run_actions(
arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()},
config_filename=flexmock(),
config={'repositories': [], 'skip_actions': ['create']},
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository={'path': 'repo'},
)
)
def test_run_actions_runs_prune(): def test_run_actions_runs_prune():
flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('add_custom_log_levels')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
@ -522,6 +550,24 @@ def test_run_actions_runs_prune():
) )
def test_run_actions_with_skip_actions_skips_prune():
flexmock(module).should_receive('add_custom_log_levels')
flexmock(module.command).should_receive('execute_hook')
flexmock(borgmatic.actions.prune).should_receive('run_prune').never()
tuple(
module.run_actions(
arguments={'global': flexmock(dry_run=False, log_file='foo'), 'prune': flexmock()},
config_filename=flexmock(),
config={'repositories': [], 'skip_actions': ['prune']},
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository={'path': 'repo'},
)
)
def test_run_actions_runs_compact(): def test_run_actions_runs_compact():
flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('add_custom_log_levels')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
@ -540,6 +586,24 @@ def test_run_actions_runs_compact():
) )
def test_run_actions_with_skip_actions_skips_compact():
flexmock(module).should_receive('add_custom_log_levels')
flexmock(module.command).should_receive('execute_hook')
flexmock(borgmatic.actions.compact).should_receive('run_compact').never()
tuple(
module.run_actions(
arguments={'global': flexmock(dry_run=False, log_file='foo'), 'compact': flexmock()},
config_filename=flexmock(),
config={'repositories': [], 'skip_actions': ['compact']},
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository={'path': 'repo'},
)
)
def test_run_actions_runs_check_when_repository_enabled_for_checks(): def test_run_actions_runs_check_when_repository_enabled_for_checks():
flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('add_custom_log_levels')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
@ -578,6 +642,25 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks():
) )
def test_run_actions_with_skip_actions_skips_check():
flexmock(module).should_receive('add_custom_log_levels')
flexmock(module.command).should_receive('execute_hook')
flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True)
flexmock(borgmatic.actions.check).should_receive('run_check').never()
tuple(
module.run_actions(
arguments={'global': flexmock(dry_run=False, log_file='foo'), 'check': flexmock()},
config_filename=flexmock(),
config={'repositories': [], 'skip_actions': ['check']},
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository={'path': 'repo'},
)
)
def test_run_actions_runs_extract(): def test_run_actions_runs_extract():
flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('add_custom_log_levels')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')

View file

@ -2,14 +2,14 @@ from borgmatic.config import checks as module
def test_repository_enabled_for_checks_defaults_to_enabled_for_all_repositories(): def test_repository_enabled_for_checks_defaults_to_enabled_for_all_repositories():
enabled = module.repository_enabled_for_checks('repo.borg', consistency={}) enabled = module.repository_enabled_for_checks('repo.borg', config={})
assert enabled assert enabled
def test_repository_enabled_for_checks_is_enabled_for_specified_repositories(): def test_repository_enabled_for_checks_is_enabled_for_specified_repositories():
enabled = module.repository_enabled_for_checks( enabled = module.repository_enabled_for_checks(
'repo.borg', consistency={'check_repositories': ['repo.borg', 'other.borg']} 'repo.borg', config={'check_repositories': ['repo.borg', 'other.borg']}
) )
assert enabled assert enabled
@ -17,7 +17,7 @@ def test_repository_enabled_for_checks_is_enabled_for_specified_repositories():
def test_repository_enabled_for_checks_is_disabled_for_other_repositories(): def test_repository_enabled_for_checks_is_disabled_for_other_repositories():
enabled = module.repository_enabled_for_checks( enabled = module.repository_enabled_for_checks(
'repo.borg', consistency={'check_repositories': ['other.borg']} 'repo.borg', config={'check_repositories': ['other.borg']}
) )
assert not enabled assert not enabled