Only check archives with matching prefix.

This commit is contained in:
Nick Whyte 2018-03-04 17:17:39 +11:00 committed by Dan Helfman
parent 0112407250
commit c64d0100d5
8 changed files with 84 additions and 13 deletions

View file

@ -8,3 +8,4 @@ newtonne: Read encryption password from external file
Robin `ypid` Schneider: Support additional options of Borg Robin `ypid` Schneider: Support additional options of Borg
Scott Squires: Custom archive names Scott Squires: Custom archive names
Thomas LÉVEIL: Support for a keep_minutely prune option Thomas LÉVEIL: Support for a keep_minutely prune option
Nick Whyte: Support prefix filtering for archive consistency checks

View file

@ -82,10 +82,13 @@ def check_archives(verbosity, repository, storage_config, consistency_config, lo
VERBOSITY_LOTS: ('--debug',), VERBOSITY_LOTS: ('--debug',),
}.get(verbosity, ()) }.get(verbosity, ())
prefix = consistency_config.get('prefix', '{hostname}-')
prefix_flags = ('--prefix', prefix) if prefix else ()
full_command = ( full_command = (
local_path, 'check', local_path, 'check',
repository, repository,
) + _make_check_flags(checks, check_last) + remote_path_flags + lock_wait_flags + verbosity_flags ) + _make_check_flags(checks, check_last) + prefix_flags + remote_path_flags + lock_wait_flags + verbosity_flags
# The check command spews to stdout/stderr even without the verbose flag. Suppress it. # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
stdout = None if verbosity_flags else open(os.devnull, 'w') stdout = None if verbosity_flags else open(os.devnull, 'w')

View file

@ -218,6 +218,13 @@ map:
desc: Restrict the number of checked archives to the last n. Applies only to the desc: Restrict the number of checked archives to the last n. Applies only to the
"archives" check. "archives" check.
example: 3 example: 3
prefix:
type: scalar
desc: |
When performing consistency checks, only consider archive names starting with
this prefix. Borg placeholders can be used. See the output of
"borg help placeholders" for details. Default is "{hostname}-".
example: sourcehostname
hooks: hooks:
desc: | desc: |
Shell commands or scripts to execute before and after a backup or if an error has occurred. Shell commands or scripts to execute before and after a backup or if an error has occurred.

View file

@ -8,6 +8,8 @@ import pykwalify.errors
from ruamel import yaml from ruamel import yaml
logger = logging.getLogger(__name__)
def schema_filename(): def schema_filename():
''' '''
Path to the installed YAML configuration schema file, used to validate and parse the Path to the installed YAML configuration schema file, used to validate and parse the
@ -50,6 +52,11 @@ def apply_logical_validation(config_filename, parsed_configuration):
) )
) )
consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix')
if archive_name_format and not consistency_prefix:
logger.warning('Since version 1.2.0, if you provide `archive_name_format`, you must also'
' specify `consistency.prefix`.')
def parse_configuration(config_filename, schema_filename): def parse_configuration(config_filename, schema_filename):
''' '''

View file

@ -89,7 +89,7 @@ def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
) )
def test_check_archives_calls_borg_with_parameters(checks): def test_check_archives_calls_borg_with_parameters(checks):
check_last = flexmock() check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock consistency_config = {'check_last': check_last}
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock() stdout = flexmock()
@ -111,7 +111,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
def test_check_archives_with_extract_check_calls_extract_only(): def test_check_archives_with_extract_check_calls_extract_only():
checks = ('extract',) checks = ('extract',)
check_last = flexmock() check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock consistency_config = {'check_last': check_last}
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').never() flexmock(module).should_receive('_make_check_flags').never()
flexmock(module.extract).should_receive('extract_last_archive_dry_run').once() flexmock(module.extract).should_receive('extract_last_archive_dry_run').once()
@ -127,7 +127,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter(): def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
checks = ('repository',) checks = ('repository',)
consistency_config = flexmock().should_receive('get').and_return(None).mock consistency_config = {'check_last': None}
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').and_return(()) flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock( insert_subprocess_mock(
@ -145,7 +145,7 @@ def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter(): def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
checks = ('repository',) checks = ('repository',)
consistency_config = flexmock().should_receive('get').and_return(None).mock consistency_config = {'check_last': None}
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').and_return(()) flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock( insert_subprocess_mock(
@ -162,7 +162,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
def test_check_archives_without_any_checks_bails(): def test_check_archives_without_any_checks_bails():
consistency_config = flexmock().should_receive('get').and_return(None).mock consistency_config = {'check_last': None}
flexmock(module).should_receive('_parse_checks').and_return(()) flexmock(module).should_receive('_parse_checks').and_return(())
insert_subprocess_never() insert_subprocess_never()
@ -177,7 +177,7 @@ def test_check_archives_without_any_checks_bails():
def test_check_archives_with_local_path_calls_borg_via_local_path(): def test_check_archives_with_local_path_calls_borg_via_local_path():
checks = ('repository',) checks = ('repository',)
check_last = flexmock() check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock consistency_config = {'check_last': check_last}
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock() stdout = flexmock()
@ -200,7 +200,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(): def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters():
checks = ('repository',) checks = ('repository',)
check_last = flexmock() check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock consistency_config = {'check_last': check_last}
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock() stdout = flexmock()
@ -223,7 +223,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
checks = ('repository',) checks = ('repository',)
check_last = flexmock() check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock consistency_config = {'check_last': check_last}
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock() stdout = flexmock()
@ -240,3 +240,26 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
storage_config={'lock_wait': 5}, storage_config={'lock_wait': 5},
consistency_config=consistency_config, consistency_config=consistency_config,
) )
def test_check_archives_with_retention_prefix():
checks = ('repository',)
check_last = flexmock()
consistency_config = {'check_last': check_last, 'prefix': 'foo-'}
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock()
insert_subprocess_mock(
('borg', 'check', 'repo', '--prefix', 'foo-'),
stdout=stdout, stderr=STDOUT,
)
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull')
module.check_archives(
verbosity=None,
repository='repo',
storage_config={},
consistency_config=consistency_config,
)

View file

@ -1,4 +1,5 @@
import pytest import pytest
from flexmock import flexmock
from borgmatic.config import validate as module from borgmatic.config import validate as module
@ -23,13 +24,42 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_
}, },
) )
def test_apply_logical_validation_raises_if_archive_name_format_present_without_retention_prefix():
with pytest.raises(module.Validation_error):
module.apply_logical_validation(
'config.yaml',
{
'storage': {'archive_name_format': '{hostname}-{now}'},
'retention': {'keep_daily': 7},
'consistency': {'prefix': '{hostname}-'}
},
)
def test_apply_logical_validation_warns_if_archive_name_format_present_without_consistency_prefix():
logger = flexmock(module.logger)
logger.should_receive('warning').once()
def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present():
module.apply_logical_validation( module.apply_logical_validation(
'config.yaml', 'config.yaml',
{ {
'storage': {'archive_name_format': '{hostname}-{now}'}, 'storage': {'archive_name_format': '{hostname}-{now}'},
'retention': {'prefix': '{hostname}-'}, 'retention': {'prefix': '{hostname}-'},
'consistency': {},
},
)
def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present():
logger = flexmock(module.logger)
logger.should_receive('warning').never()
module.apply_logical_validation(
'config.yaml',
{
'storage': {'archive_name_format': '{hostname}-{now}'},
'retention': {'prefix': '{hostname}-'},
'consistency': {'prefix': '{hostname}-'}
}, },
) )

View file

@ -7,5 +7,5 @@ def test_verbosity_to_log_level_maps_known_verbosity_to_log_level():
assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO
def test_verbosity_to_log_level_maps_unknown_verbosity_to_error_level(): def test_verbosity_to_log_level_maps_unknown_verbosity_to_warning_level():
assert module.verbosity_to_log_level('my pants') == logging.ERROR assert module.verbosity_to_log_level('my pants') == logging.WARNING

View file

@ -12,4 +12,4 @@ def verbosity_to_log_level(verbosity):
return { return {
VERBOSITY_SOME: logging.INFO, VERBOSITY_SOME: logging.INFO,
VERBOSITY_LOTS: logging.DEBUG, VERBOSITY_LOTS: logging.DEBUG,
}.get(verbosity, logging.ERROR) }.get(verbosity, logging.WARNING)