#1: Add support for "borg check --last N" to Borg backend.
This commit is contained in:
parent
f5e0e10143
commit
9c06874073
9 changed files with 94 additions and 57 deletions
4
NEWS
4
NEWS
|
@ -1,3 +1,7 @@
|
||||||
|
0.1.3
|
||||||
|
|
||||||
|
* #1: Add support for "borg check --last N" to Borg backend.
|
||||||
|
|
||||||
0.1.2
|
0.1.2
|
||||||
|
|
||||||
* As a convenience to new users, allow a missing default excludes file.
|
* As a convenience to new users, allow a missing default excludes file.
|
||||||
|
|
|
@ -5,7 +5,7 @@ from atticmatic.backends import shared
|
||||||
# An atticmatic backend that supports Attic for actually handling backups.
|
# An atticmatic backend that supports Attic for actually handling backups.
|
||||||
|
|
||||||
COMMAND = 'attic'
|
COMMAND = 'attic'
|
||||||
|
CONFIG_FORMAT = shared.CONFIG_FORMAT
|
||||||
|
|
||||||
create_archive = partial(shared.create_archive, command=COMMAND)
|
create_archive = partial(shared.create_archive, command=COMMAND)
|
||||||
prune_archives = partial(shared.prune_archives, command=COMMAND)
|
prune_archives = partial(shared.prune_archives, command=COMMAND)
|
||||||
|
|
|
@ -1,10 +1,22 @@
|
||||||
from functools import partial
|
from functools import partial
|
||||||
|
|
||||||
|
from atticmatic.config import Section_format, option
|
||||||
from atticmatic.backends import shared
|
from atticmatic.backends import shared
|
||||||
|
|
||||||
# An atticmatic backend that supports Borg for actually handling backups.
|
# An atticmatic backend that supports Borg for actually handling backups.
|
||||||
|
|
||||||
COMMAND = 'borg'
|
COMMAND = 'borg'
|
||||||
|
CONFIG_FORMAT = (
|
||||||
|
shared.CONFIG_FORMAT[0], # location
|
||||||
|
shared.CONFIG_FORMAT[1], # retention
|
||||||
|
Section_format(
|
||||||
|
'consistency',
|
||||||
|
(
|
||||||
|
option('checks', required=False),
|
||||||
|
option('check_last', required=False),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
create_archive = partial(shared.create_archive, command=COMMAND)
|
create_archive = partial(shared.create_archive, command=COMMAND)
|
||||||
|
|
|
@ -3,6 +3,7 @@ import os
|
||||||
import platform
|
import platform
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
from atticmatic.config import Section_format, option
|
||||||
from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
|
from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
|
||||||
|
|
||||||
|
|
||||||
|
@ -12,6 +13,34 @@ from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
|
||||||
# atticmatic.backends.borg.
|
# atticmatic.backends.borg.
|
||||||
|
|
||||||
|
|
||||||
|
CONFIG_FORMAT = (
|
||||||
|
Section_format(
|
||||||
|
'location',
|
||||||
|
(
|
||||||
|
option('source_directories'),
|
||||||
|
option('repository'),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
Section_format(
|
||||||
|
'retention',
|
||||||
|
(
|
||||||
|
option('keep_within', required=False),
|
||||||
|
option('keep_hourly', int, required=False),
|
||||||
|
option('keep_daily', int, required=False),
|
||||||
|
option('keep_weekly', int, required=False),
|
||||||
|
option('keep_monthly', int, required=False),
|
||||||
|
option('keep_yearly', int, required=False),
|
||||||
|
option('prefix', required=False),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
Section_format(
|
||||||
|
'consistency',
|
||||||
|
(
|
||||||
|
option('checks', required=False),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
def create_archive(excludes_filename, verbosity, source_directories, repository, command):
|
def create_archive(excludes_filename, verbosity, source_directories, repository, command):
|
||||||
'''
|
'''
|
||||||
Given an excludes filename (or None), a vebosity flag, a space-separated list of source
|
Given an excludes filename (or None), a vebosity flag, a space-separated list of source
|
||||||
|
@ -110,7 +139,7 @@ def _parse_checks(consistency_config):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _make_check_flags(checks):
|
def _make_check_flags(checks, check_last=None):
|
||||||
'''
|
'''
|
||||||
Given a parsed sequence of checks, transform it into tuple of command-line flags.
|
Given a parsed sequence of checks, transform it into tuple of command-line flags.
|
||||||
|
|
||||||
|
@ -121,13 +150,17 @@ def _make_check_flags(checks):
|
||||||
This will be returned as:
|
This will be returned as:
|
||||||
|
|
||||||
('--repository-only',)
|
('--repository-only',)
|
||||||
|
|
||||||
|
Additionally, if a check_last value is given, a "--last" flag will be added. Note that only
|
||||||
|
Borg supports this flag.
|
||||||
'''
|
'''
|
||||||
|
last_flag = ('--last', check_last) if check_last else ()
|
||||||
if checks == DEFAULT_CHECKS:
|
if checks == DEFAULT_CHECKS:
|
||||||
return ()
|
return last_flag
|
||||||
|
|
||||||
return tuple(
|
return tuple(
|
||||||
'--{}-only'.format(check) for check in checks
|
'--{}-only'.format(check) for check in checks
|
||||||
)
|
) + last_flag
|
||||||
|
|
||||||
|
|
||||||
def check_archives(verbosity, repository, consistency_config, command):
|
def check_archives(verbosity, repository, consistency_config, command):
|
||||||
|
@ -138,6 +171,7 @@ def check_archives(verbosity, repository, consistency_config, command):
|
||||||
If there are no consistency checks to run, skip running them.
|
If there are no consistency checks to run, skip running them.
|
||||||
'''
|
'''
|
||||||
checks = _parse_checks(consistency_config)
|
checks = _parse_checks(consistency_config)
|
||||||
|
check_last = consistency_config.get('check_last', None)
|
||||||
if not checks:
|
if not checks:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -149,7 +183,7 @@ def check_archives(verbosity, repository, consistency_config, command):
|
||||||
full_command = (
|
full_command = (
|
||||||
command, 'check',
|
command, 'check',
|
||||||
repository,
|
repository,
|
||||||
) + _make_check_flags(checks) + verbosity_flags
|
) + _make_check_flags(checks, check_last) + verbosity_flags
|
||||||
|
|
||||||
# The check command spews to stdout even without the verbose flag. Suppress it.
|
# The check command spews to stdout even without the verbose flag. Suppress it.
|
||||||
stdout = None if verbosity_flags else open(os.devnull, 'w')
|
stdout = None if verbosity_flags else open(os.devnull, 'w')
|
||||||
|
|
|
@ -60,9 +60,9 @@ def main():
|
||||||
try:
|
try:
|
||||||
command_name = os.path.basename(sys.argv[0])
|
command_name = os.path.basename(sys.argv[0])
|
||||||
args = parse_arguments(command_name, *sys.argv[1:])
|
args = parse_arguments(command_name, *sys.argv[1:])
|
||||||
config = parse_configuration(args.config_filename)
|
|
||||||
repository = config.location['repository']
|
|
||||||
backend = load_backend(command_name)
|
backend = load_backend(command_name)
|
||||||
|
config = parse_configuration(args.config_filename, backend.CONFIG_FORMAT)
|
||||||
|
repository = config.location['repository']
|
||||||
|
|
||||||
backend.create_archive(args.excludes_filename, args.verbosity, **config.location)
|
backend.create_archive(args.excludes_filename, args.verbosity, **config.location)
|
||||||
backend.prune_archives(args.verbosity, repository, config.retention)
|
backend.prune_archives(args.verbosity, repository, config.retention)
|
||||||
|
|
|
@ -20,35 +20,6 @@ def option(name, value_type=str, required=True):
|
||||||
return Config_option(name, value_type, required)
|
return Config_option(name, value_type, required)
|
||||||
|
|
||||||
|
|
||||||
CONFIG_FORMAT = (
|
|
||||||
Section_format(
|
|
||||||
'location',
|
|
||||||
(
|
|
||||||
option('source_directories'),
|
|
||||||
option('repository'),
|
|
||||||
),
|
|
||||||
),
|
|
||||||
Section_format(
|
|
||||||
'retention',
|
|
||||||
(
|
|
||||||
option('keep_within', required=False),
|
|
||||||
option('keep_hourly', int, required=False),
|
|
||||||
option('keep_daily', int, required=False),
|
|
||||||
option('keep_weekly', int, required=False),
|
|
||||||
option('keep_monthly', int, required=False),
|
|
||||||
option('keep_yearly', int, required=False),
|
|
||||||
option('prefix', required=False),
|
|
||||||
),
|
|
||||||
),
|
|
||||||
Section_format(
|
|
||||||
'consistency',
|
|
||||||
(
|
|
||||||
option('checks', required=False),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def validate_configuration_format(parser, config_format):
|
def validate_configuration_format(parser, config_format):
|
||||||
'''
|
'''
|
||||||
Given an open ConfigParser and an expected config file format, validate that the parsed
|
Given an open ConfigParser and an expected config file format, validate that the parsed
|
||||||
|
@ -110,11 +81,6 @@ def validate_configuration_format(parser, config_format):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# Describes a parsed configuration, where each attribute is the name of a configuration file section
|
|
||||||
# and each value is a dict of that section's parsed options.
|
|
||||||
Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT))
|
|
||||||
|
|
||||||
|
|
||||||
def parse_section_options(parser, section_format):
|
def parse_section_options(parser, section_format):
|
||||||
'''
|
'''
|
||||||
Given an open ConfigParser and an expected section format, return the option values from that
|
Given an open ConfigParser and an expected section format, return the option values from that
|
||||||
|
@ -135,21 +101,25 @@ def parse_section_options(parser, section_format):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def parse_configuration(config_filename):
|
def parse_configuration(config_filename, config_format):
|
||||||
'''
|
'''
|
||||||
Given a config filename of the expected format, return the parsed configuration as Parsed_config
|
Given a config filename and an expected config file format, return the parsed configuration
|
||||||
data structure.
|
as a namedtuple with one attribute for each parsed section.
|
||||||
|
|
||||||
Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
|
Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
|
||||||
'''
|
'''
|
||||||
parser = ConfigParser()
|
parser = ConfigParser()
|
||||||
parser.read(config_filename)
|
parser.read(config_filename)
|
||||||
|
|
||||||
validate_configuration_format(parser, CONFIG_FORMAT)
|
validate_configuration_format(parser, config_format)
|
||||||
|
|
||||||
|
# Describes a parsed configuration, where each attribute is the name of a configuration file
|
||||||
|
# section and each value is a dict of that section's parsed options.
|
||||||
|
Parsed_config = namedtuple('Parsed_config', (section_format.name for section_format in config_format))
|
||||||
|
|
||||||
return Parsed_config(
|
return Parsed_config(
|
||||||
*(
|
*(
|
||||||
parse_section_options(parser, section_format)
|
parse_section_options(parser, section_format)
|
||||||
for section_format in CONFIG_FORMAT
|
for section_format in config_format
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
|
@ -196,10 +196,24 @@ def test_make_check_flags_with_default_checks_returns_no_flags():
|
||||||
assert flags == ()
|
assert flags == ()
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
|
||||||
|
flags = module._make_check_flags(('foo', 'bar'), check_last=3)
|
||||||
|
|
||||||
|
assert flags == ('--foo-only', '--bar-only', '--last', 3)
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_check_flags_with_last_returns_last_flag():
|
||||||
|
flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
|
||||||
|
|
||||||
|
assert flags == ('--last', 3)
|
||||||
|
|
||||||
|
|
||||||
def test_check_archives_should_call_attic_with_parameters():
|
def test_check_archives_should_call_attic_with_parameters():
|
||||||
consistency_config = flexmock()
|
checks = flexmock()
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
|
check_last = flexmock()
|
||||||
flexmock(module).should_receive('_make_check_flags').and_return(())
|
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
||||||
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
|
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
|
||||||
stdout = flexmock()
|
stdout = flexmock()
|
||||||
insert_subprocess_mock(
|
insert_subprocess_mock(
|
||||||
('attic', 'check', 'repo'),
|
('attic', 'check', 'repo'),
|
||||||
|
@ -219,7 +233,7 @@ def test_check_archives_should_call_attic_with_parameters():
|
||||||
|
|
||||||
|
|
||||||
def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
|
def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
|
||||||
consistency_config = flexmock()
|
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
|
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
|
||||||
flexmock(module).should_receive('_make_check_flags').and_return(())
|
flexmock(module).should_receive('_make_check_flags').and_return(())
|
||||||
insert_subprocess_mock(
|
insert_subprocess_mock(
|
||||||
|
@ -238,7 +252,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param
|
||||||
|
|
||||||
|
|
||||||
def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
|
def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
|
||||||
consistency_config = flexmock()
|
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
|
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
|
||||||
flexmock(module).should_receive('_make_check_flags').and_return(())
|
flexmock(module).should_receive('_make_check_flags').and_return(())
|
||||||
insert_subprocess_mock(
|
insert_subprocess_mock(
|
||||||
|
@ -257,7 +271,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param
|
||||||
|
|
||||||
|
|
||||||
def test_check_archives_without_any_checks_should_bail():
|
def test_check_archives_without_any_checks_should_bail():
|
||||||
consistency_config = flexmock()
|
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(())
|
flexmock(module).should_receive('_parse_checks').and_return(())
|
||||||
insert_subprocess_never()
|
insert_subprocess_never()
|
||||||
|
|
||||||
|
|
|
@ -205,17 +205,18 @@ def insert_mock_parser():
|
||||||
|
|
||||||
def test_parse_configuration_should_return_section_configs():
|
def test_parse_configuration_should_return_section_configs():
|
||||||
parser = insert_mock_parser()
|
parser = insert_mock_parser()
|
||||||
|
config_format = (flexmock(name='items'), flexmock(name='things'))
|
||||||
mock_module = flexmock(module)
|
mock_module = flexmock(module)
|
||||||
mock_module.should_receive('validate_configuration_format').with_args(
|
mock_module.should_receive('validate_configuration_format').with_args(
|
||||||
parser, module.CONFIG_FORMAT,
|
parser, config_format,
|
||||||
).once()
|
).once()
|
||||||
mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT)
|
mock_section_configs = (flexmock(), flexmock())
|
||||||
|
|
||||||
for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs):
|
for section_format, section_config in zip(config_format, mock_section_configs):
|
||||||
mock_module.should_receive('parse_section_options').with_args(
|
mock_module.should_receive('parse_section_options').with_args(
|
||||||
parser, section_format,
|
parser, section_format,
|
||||||
).and_return(section_config).once()
|
).and_return(section_config).once()
|
||||||
|
|
||||||
parsed_config = module.parse_configuration('filename')
|
parsed_config = module.parse_configuration('filename', config_format)
|
||||||
|
|
||||||
assert parsed_config == module.Parsed_config(*mock_section_configs)
|
assert parsed_config == type(parsed_config)(*mock_section_configs)
|
||||||
|
|
|
@ -23,3 +23,5 @@ keep_yearly: 1
|
||||||
# checks. See https://attic-backup.org/usage.html#attic-check or
|
# checks. See https://attic-backup.org/usage.html#attic-check or
|
||||||
# https://borgbackup.github.io/borgbackup/usage.html#borg-check for details.
|
# https://borgbackup.github.io/borgbackup/usage.html#borg-check for details.
|
||||||
checks: repository archives
|
checks: repository archives
|
||||||
|
# For Borg only, you can restrict the number of checked archives to the last n.
|
||||||
|
#check_last: 3
|
||||||
|
|
Loading…
Reference in a new issue