#1: Add support for "borg check --last N" to Borg backend.

This commit is contained in:
Dan Helfman 2015-07-27 21:47:52 -07:00
parent f5e0e10143
commit 9c06874073
9 changed files with 94 additions and 57 deletions

4
NEWS
View file

@ -1,3 +1,7 @@
0.1.3
* #1: Add support for "borg check --last N" to Borg backend.
0.1.2 0.1.2
* As a convenience to new users, allow a missing default excludes file. * As a convenience to new users, allow a missing default excludes file.

View file

@ -5,7 +5,7 @@ from atticmatic.backends import shared
# An atticmatic backend that supports Attic for actually handling backups. # An atticmatic backend that supports Attic for actually handling backups.
COMMAND = 'attic' COMMAND = 'attic'
CONFIG_FORMAT = shared.CONFIG_FORMAT
create_archive = partial(shared.create_archive, command=COMMAND) create_archive = partial(shared.create_archive, command=COMMAND)
prune_archives = partial(shared.prune_archives, command=COMMAND) prune_archives = partial(shared.prune_archives, command=COMMAND)

View file

@ -1,10 +1,22 @@
from functools import partial from functools import partial
from atticmatic.config import Section_format, option
from atticmatic.backends import shared from atticmatic.backends import shared
# An atticmatic backend that supports Borg for actually handling backups. # An atticmatic backend that supports Borg for actually handling backups.
COMMAND = 'borg' COMMAND = 'borg'
CONFIG_FORMAT = (
shared.CONFIG_FORMAT[0], # location
shared.CONFIG_FORMAT[1], # retention
Section_format(
'consistency',
(
option('checks', required=False),
option('check_last', required=False),
),
)
)
create_archive = partial(shared.create_archive, command=COMMAND) create_archive = partial(shared.create_archive, command=COMMAND)

View file

@ -3,6 +3,7 @@ import os
import platform import platform
import subprocess import subprocess
from atticmatic.config import Section_format, option
from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
@ -12,6 +13,34 @@ from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
# atticmatic.backends.borg. # atticmatic.backends.borg.
CONFIG_FORMAT = (
Section_format(
'location',
(
option('source_directories'),
option('repository'),
),
),
Section_format(
'retention',
(
option('keep_within', required=False),
option('keep_hourly', int, required=False),
option('keep_daily', int, required=False),
option('keep_weekly', int, required=False),
option('keep_monthly', int, required=False),
option('keep_yearly', int, required=False),
option('prefix', required=False),
),
),
Section_format(
'consistency',
(
option('checks', required=False),
),
)
)
def create_archive(excludes_filename, verbosity, source_directories, repository, command): def create_archive(excludes_filename, verbosity, source_directories, repository, command):
''' '''
Given an excludes filename (or None), a vebosity flag, a space-separated list of source Given an excludes filename (or None), a vebosity flag, a space-separated list of source
@ -110,7 +139,7 @@ def _parse_checks(consistency_config):
) )
def _make_check_flags(checks): def _make_check_flags(checks, check_last=None):
''' '''
Given a parsed sequence of checks, transform it into tuple of command-line flags. Given a parsed sequence of checks, transform it into tuple of command-line flags.
@ -121,13 +150,17 @@ def _make_check_flags(checks):
This will be returned as: This will be returned as:
('--repository-only',) ('--repository-only',)
Additionally, if a check_last value is given, a "--last" flag will be added. Note that only
Borg supports this flag.
''' '''
last_flag = ('--last', check_last) if check_last else ()
if checks == DEFAULT_CHECKS: if checks == DEFAULT_CHECKS:
return () return last_flag
return tuple( return tuple(
'--{}-only'.format(check) for check in checks '--{}-only'.format(check) for check in checks
) ) + last_flag
def check_archives(verbosity, repository, consistency_config, command): def check_archives(verbosity, repository, consistency_config, command):
@ -138,6 +171,7 @@ def check_archives(verbosity, repository, consistency_config, command):
If there are no consistency checks to run, skip running them. If there are no consistency checks to run, skip running them.
''' '''
checks = _parse_checks(consistency_config) checks = _parse_checks(consistency_config)
check_last = consistency_config.get('check_last', None)
if not checks: if not checks:
return return
@ -149,7 +183,7 @@ def check_archives(verbosity, repository, consistency_config, command):
full_command = ( full_command = (
command, 'check', command, 'check',
repository, repository,
) + _make_check_flags(checks) + verbosity_flags ) + _make_check_flags(checks, check_last) + verbosity_flags
# The check command spews to stdout even without the verbose flag. Suppress it. # The check command spews to stdout even without the verbose flag. Suppress it.
stdout = None if verbosity_flags else open(os.devnull, 'w') stdout = None if verbosity_flags else open(os.devnull, 'w')

View file

@ -60,9 +60,9 @@ def main():
try: try:
command_name = os.path.basename(sys.argv[0]) command_name = os.path.basename(sys.argv[0])
args = parse_arguments(command_name, *sys.argv[1:]) args = parse_arguments(command_name, *sys.argv[1:])
config = parse_configuration(args.config_filename)
repository = config.location['repository']
backend = load_backend(command_name) backend = load_backend(command_name)
config = parse_configuration(args.config_filename, backend.CONFIG_FORMAT)
repository = config.location['repository']
backend.create_archive(args.excludes_filename, args.verbosity, **config.location) backend.create_archive(args.excludes_filename, args.verbosity, **config.location)
backend.prune_archives(args.verbosity, repository, config.retention) backend.prune_archives(args.verbosity, repository, config.retention)

View file

@ -20,35 +20,6 @@ def option(name, value_type=str, required=True):
return Config_option(name, value_type, required) return Config_option(name, value_type, required)
CONFIG_FORMAT = (
Section_format(
'location',
(
option('source_directories'),
option('repository'),
),
),
Section_format(
'retention',
(
option('keep_within', required=False),
option('keep_hourly', int, required=False),
option('keep_daily', int, required=False),
option('keep_weekly', int, required=False),
option('keep_monthly', int, required=False),
option('keep_yearly', int, required=False),
option('prefix', required=False),
),
),
Section_format(
'consistency',
(
option('checks', required=False),
),
)
)
def validate_configuration_format(parser, config_format): def validate_configuration_format(parser, config_format):
''' '''
Given an open ConfigParser and an expected config file format, validate that the parsed Given an open ConfigParser and an expected config file format, validate that the parsed
@ -110,11 +81,6 @@ def validate_configuration_format(parser, config_format):
) )
# Describes a parsed configuration, where each attribute is the name of a configuration file section
# and each value is a dict of that section's parsed options.
Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT))
def parse_section_options(parser, section_format): def parse_section_options(parser, section_format):
''' '''
Given an open ConfigParser and an expected section format, return the option values from that Given an open ConfigParser and an expected section format, return the option values from that
@ -135,21 +101,25 @@ def parse_section_options(parser, section_format):
) )
def parse_configuration(config_filename): def parse_configuration(config_filename, config_format):
''' '''
Given a config filename of the expected format, return the parsed configuration as Parsed_config Given a config filename and an expected config file format, return the parsed configuration
data structure. as a namedtuple with one attribute for each parsed section.
Raise IOError if the file cannot be read, or ValueError if the format is not as expected. Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
''' '''
parser = ConfigParser() parser = ConfigParser()
parser.read(config_filename) parser.read(config_filename)
validate_configuration_format(parser, CONFIG_FORMAT) validate_configuration_format(parser, config_format)
# Describes a parsed configuration, where each attribute is the name of a configuration file
# section and each value is a dict of that section's parsed options.
Parsed_config = namedtuple('Parsed_config', (section_format.name for section_format in config_format))
return Parsed_config( return Parsed_config(
*( *(
parse_section_options(parser, section_format) parse_section_options(parser, section_format)
for section_format in CONFIG_FORMAT for section_format in config_format
) )
) )

View file

@ -196,10 +196,24 @@ def test_make_check_flags_with_default_checks_returns_no_flags():
assert flags == () assert flags == ()
def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
flags = module._make_check_flags(('foo', 'bar'), check_last=3)
assert flags == ('--foo-only', '--bar-only', '--last', 3)
def test_make_check_flags_with_last_returns_last_flag():
flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
assert flags == ('--last', 3)
def test_check_archives_should_call_attic_with_parameters(): def test_check_archives_should_call_attic_with_parameters():
consistency_config = flexmock() checks = flexmock()
flexmock(module).should_receive('_parse_checks').and_return(flexmock()) check_last = flexmock()
flexmock(module).should_receive('_make_check_flags').and_return(()) consistency_config = flexmock().should_receive('get').and_return(check_last).mock
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock() stdout = flexmock()
insert_subprocess_mock( insert_subprocess_mock(
('attic', 'check', 'repo'), ('attic', 'check', 'repo'),
@ -219,7 +233,7 @@ def test_check_archives_should_call_attic_with_parameters():
def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter(): def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
consistency_config = flexmock() consistency_config = flexmock().should_receive('get').and_return(None).mock
flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_parse_checks').and_return(flexmock())
flexmock(module).should_receive('_make_check_flags').and_return(()) flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock( insert_subprocess_mock(
@ -238,7 +252,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param
def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter(): def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
consistency_config = flexmock() consistency_config = flexmock().should_receive('get').and_return(None).mock
flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_parse_checks').and_return(flexmock())
flexmock(module).should_receive('_make_check_flags').and_return(()) flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock( insert_subprocess_mock(
@ -257,7 +271,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param
def test_check_archives_without_any_checks_should_bail(): def test_check_archives_without_any_checks_should_bail():
consistency_config = flexmock() consistency_config = flexmock().should_receive('get').and_return(None).mock
flexmock(module).should_receive('_parse_checks').and_return(()) flexmock(module).should_receive('_parse_checks').and_return(())
insert_subprocess_never() insert_subprocess_never()

View file

@ -205,17 +205,18 @@ def insert_mock_parser():
def test_parse_configuration_should_return_section_configs(): def test_parse_configuration_should_return_section_configs():
parser = insert_mock_parser() parser = insert_mock_parser()
config_format = (flexmock(name='items'), flexmock(name='things'))
mock_module = flexmock(module) mock_module = flexmock(module)
mock_module.should_receive('validate_configuration_format').with_args( mock_module.should_receive('validate_configuration_format').with_args(
parser, module.CONFIG_FORMAT, parser, config_format,
).once() ).once()
mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT) mock_section_configs = (flexmock(), flexmock())
for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs): for section_format, section_config in zip(config_format, mock_section_configs):
mock_module.should_receive('parse_section_options').with_args( mock_module.should_receive('parse_section_options').with_args(
parser, section_format, parser, section_format,
).and_return(section_config).once() ).and_return(section_config).once()
parsed_config = module.parse_configuration('filename') parsed_config = module.parse_configuration('filename', config_format)
assert parsed_config == module.Parsed_config(*mock_section_configs) assert parsed_config == type(parsed_config)(*mock_section_configs)

View file

@ -23,3 +23,5 @@ keep_yearly: 1
# checks. See https://attic-backup.org/usage.html#attic-check or # checks. See https://attic-backup.org/usage.html#attic-check or
# https://borgbackup.github.io/borgbackup/usage.html#borg-check for details. # https://borgbackup.github.io/borgbackup/usage.html#borg-check for details.
checks: repository archives checks: repository archives
# For Borg only, you can restrict the number of checked archives to the last n.
#check_last: 3