Support for backing up to multiple repositories.

This commit is contained in:
Dan Helfman 2017-07-22 22:56:46 -07:00
parent 548212274f
commit 499f8aa0a4
9 changed files with 115 additions and 63 deletions

1
NEWS
View file

@ -5,6 +5,7 @@
* Dropped Python 2 support. Now Python 3 only. * Dropped Python 2 support. Now Python 3 only.
* #18: Fix for README mention of sample files not included in package. * #18: Fix for README mention of sample files not included in package.
* #22: Sample files for triggering borgmatic from a systemd timer. * #22: Sample files for triggering borgmatic from a systemd timer.
* Support for backing up to multiple repositories.
* To free up space, now pruning backups prior to creating a new backup. * To free up space, now pruning backups prior to creating a new backup.
* Enabled test coverage output during tox runs. * Enabled test coverage output during tox runs.
* Added logo. * Added logo.

View file

@ -21,8 +21,9 @@ location:
- /etc - /etc
- /var/log/syslog* - /var/log/syslog*
# Path to local or remote repository. # Paths to local or remote repositories.
repository: user@backupserver:sourcehostname.borg repositories:
- user@backupserver:sourcehostname.borg
# Any paths matching these patterns are excluded from backups. # Any paths matching these patterns are excluded from backups.
exclude_patterns: exclude_patterns:

View file

@ -39,8 +39,7 @@ def _write_exclude_file(exclude_patterns=None):
def create_archive( def create_archive(
verbosity, storage_config, source_directories, repository, exclude_patterns=None, verbosity, repository, location_config, storage_config, command=COMMAND,
command=COMMAND, one_file_system=None, remote_path=None,
): ):
''' '''
Given a vebosity flag, a storage config dict, a list of source directories, a local or remote Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
@ -49,17 +48,18 @@ def create_archive(
sources = tuple( sources = tuple(
itertools.chain.from_iterable( itertools.chain.from_iterable(
glob.glob(directory) or [directory] glob.glob(directory) or [directory]
for directory in source_directories for directory in location_config['source_directories']
) )
) )
exclude_file = _write_exclude_file(exclude_patterns) exclude_file = _write_exclude_file(location_config.get('exclude_patterns'))
exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else () exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
compression = storage_config.get('compression', None) compression = storage_config.get('compression', None)
compression_flags = ('--compression', compression) if compression else () compression_flags = ('--compression', compression) if compression else ()
umask = storage_config.get('umask', None) umask = storage_config.get('umask', None)
umask_flags = ('--umask', str(umask)) if umask else () umask_flags = ('--umask', str(umask)) if umask else ()
one_file_system_flags = ('--one-file-system',) if one_file_system else () one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
remote_path = location_config.get('remote_path')
remote_path_flags = ('--remote-path', remote_path) if remote_path else () remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
verbosity_flags = { verbosity_flags = {
VERBOSITY_SOME: ('--info', '--stats',), VERBOSITY_SOME: ('--info', '--stats',),

View file

@ -44,17 +44,23 @@ def main(): # pragma: no cover
args = parse_arguments(*sys.argv[1:]) args = parse_arguments(*sys.argv[1:])
convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename) convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename)
config = validate.parse_configuration(args.config_filename, validate.schema_filename()) config = validate.parse_configuration(args.config_filename, validate.schema_filename())
repository = config['location']['repository'] (location, storage, retention, consistency) = (
remote_path = config['location']['remote_path']
(storage, retention, consistency) = (
config.get(section_name, {}) config.get(section_name, {})
for section_name in ('storage', 'retention', 'consistency') for section_name in ('location', 'storage', 'retention', 'consistency')
) )
remote_path = location.get('remote_path')
borg.initialize(storage) borg.initialize(storage)
borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
borg.create_archive(args.verbosity, storage, **config['location']) for repository in location['repositories']:
borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
borg.create_archive(
args.verbosity,
repository,
location,
storage,
)
borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
except (ValueError, OSError, CalledProcessError) as error: except (ValueError, OSError, CalledProcessError) as error:
print(error, file=sys.stderr) print(error, file=sys.stderr)
sys.exit(1) sys.exit(1)

View file

@ -32,9 +32,12 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
for section_name, section_config in source_config._asdict().items() for section_name, section_config in source_config._asdict().items()
]) ])
# Split space-seperated values into actual lists, and merge in excludes. # Split space-seperated values into actual lists, make "repository" into a list, and merge in
destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ') # excludes.
destination_config['location']['exclude_patterns'] = source_excludes location = destination_config['location']
location['source_directories'] = source_config.location['source_directories'].split(' ')
location['repositories'] = [location.pop('repository')]
location['exclude_patterns'] = source_excludes
if source_config.consistency['checks']: if source_config.consistency['checks']:
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')

View file

@ -25,11 +25,15 @@ map:
type: scalar type: scalar
desc: Alternate Borg remote executable. Defaults to "borg". desc: Alternate Borg remote executable. Defaults to "borg".
example: borg1 example: borg1
repository: repositories:
required: True required: True
type: scalar seq:
desc: Path to local or remote repository (required). - type: scalar
example: user@backupserver:sourcehostname.borg desc: |
Paths to local or remote repositories (required). Multiple repositories are
backed up to in sequence.
example:
- user@backupserver:sourcehostname.borg
exclude_patterns: exclude_patterns:
seq: seq:
- type: scalar - type: scalar

View file

@ -35,7 +35,8 @@ def test_parse_configuration_transforms_file_into_mapping():
- /home - /home
- /etc - /etc
repository: hostname.borg repositories:
- hostname.borg
retention: retention:
keep_daily: 7 keep_daily: 7
@ -50,7 +51,7 @@ def test_parse_configuration_transforms_file_into_mapping():
result = module.parse_configuration('config.yaml', 'schema.yaml') result = module.parse_configuration('config.yaml', 'schema.yaml')
assert result == { assert result == {
'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'}, 'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']},
'retention': {'keep_daily': 7}, 'retention': {'keep_daily': 7},
'consistency': {'checks': ['repository', 'archives']}, 'consistency': {'checks': ['repository', 'archives']},
} }
@ -65,7 +66,8 @@ def test_parse_configuration_passes_through_quoted_punctuation():
source_directories: source_directories:
- /home - /home
repository: "{}.borg" repositories:
- "{}.borg"
'''.format(escaped_punctuation) '''.format(escaped_punctuation)
) )
@ -74,7 +76,7 @@ def test_parse_configuration_passes_through_quoted_punctuation():
assert result == { assert result == {
'location': { 'location': {
'source_directories': ['/home'], 'source_directories': ['/home'],
'repository': '{}.borg'.format(string.punctuation), 'repositories': ['{}.borg'.format(string.punctuation)],
}, },
} }
@ -105,7 +107,8 @@ def test_parse_configuration_raises_for_validation_error():
''' '''
location: location:
source_directories: yes source_directories: yes
repository: hostname.borg repositories:
- hostname.borg
''' '''
) )

View file

@ -28,7 +28,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
'location', 'location',
OrderedDict([ OrderedDict([
('source_directories', ['/home']), ('source_directories', ['/home']),
('repository', 'hostname.borg'), ('repositories', ['hostname.borg']),
('exclude_patterns', ['/var']), ('exclude_patterns', ['/var']),
]), ]),
), ),
@ -41,7 +41,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
def test_convert_legacy_parsed_config_splits_space_separated_values(): def test_convert_legacy_parsed_config_splits_space_separated_values():
flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
source_config = Parsed_config( source_config = Parsed_config(
location=OrderedDict([('source_directories', '/home /etc')]), location=OrderedDict([('source_directories', '/home /etc'), ('repository', 'hostname.borg')]),
storage=OrderedDict(), storage=OrderedDict(),
retention=OrderedDict(), retention=OrderedDict(),
consistency=OrderedDict([('checks', 'repository archives')]), consistency=OrderedDict([('checks', 'repository archives')]),
@ -56,6 +56,7 @@ def test_convert_legacy_parsed_config_splits_space_separated_values():
'location', 'location',
OrderedDict([ OrderedDict([
('source_directories', ['/home', '/etc']), ('source_directories', ['/home', '/etc']),
('repositories', ['hostname.borg']),
('exclude_patterns', ['/var']), ('exclude_patterns', ['/var']),
]), ]),
), ),

View file

@ -77,11 +77,14 @@ def test_create_archive_should_call_borg_with_parameters():
insert_datetime_mock() insert_datetime_mock()
module.create_archive( module.create_archive(
exclude_patterns=None,
verbosity=None, verbosity=None,
storage_config={},
source_directories=['foo', 'bar'],
repository='repo', repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
command='borg', command='borg',
) )
@ -93,11 +96,14 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
insert_datetime_mock() insert_datetime_mock()
module.create_archive( module.create_archive(
exclude_patterns=['exclude'],
verbosity=None, verbosity=None,
storage_config={},
source_directories=['foo', 'bar'],
repository='repo', repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': ['exclude'],
},
storage_config={},
command='borg', command='borg',
) )
@ -109,11 +115,14 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter
insert_datetime_mock() insert_datetime_mock()
module.create_archive( module.create_archive(
exclude_patterns=None,
verbosity=VERBOSITY_SOME, verbosity=VERBOSITY_SOME,
storage_config={},
source_directories=['foo', 'bar'],
repository='repo', repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
command='borg', command='borg',
) )
@ -125,11 +134,14 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete
insert_datetime_mock() insert_datetime_mock()
module.create_archive( module.create_archive(
exclude_patterns=None,
verbosity=VERBOSITY_LOTS, verbosity=VERBOSITY_LOTS,
storage_config={},
source_directories=['foo', 'bar'],
repository='repo', repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
command='borg', command='borg',
) )
@ -141,11 +153,14 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param
insert_datetime_mock() insert_datetime_mock()
module.create_archive( module.create_archive(
exclude_patterns=None,
verbosity=None, verbosity=None,
storage_config={'compression': 'rle'},
source_directories=['foo', 'bar'],
repository='repo', repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={'compression': 'rle'},
command='borg', command='borg',
) )
@ -157,13 +172,16 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst
insert_datetime_mock() insert_datetime_mock()
module.create_archive( module.create_archive(
exclude_patterns=None,
verbosity=None, verbosity=None,
storage_config={},
source_directories=['foo', 'bar'],
repository='repo', repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'one_file_system': True,
'exclude_patterns': None,
},
storage_config={},
command='borg', command='borg',
one_file_system=True,
) )
@ -174,13 +192,16 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param
insert_datetime_mock() insert_datetime_mock()
module.create_archive( module.create_archive(
exclude_patterns=None,
verbosity=None, verbosity=None,
storage_config={},
source_directories=['foo', 'bar'],
repository='repo', repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'remote_path': 'borg1',
'exclude_patterns': None,
},
storage_config={},
command='borg', command='borg',
remote_path='borg1',
) )
@ -191,11 +212,14 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
insert_datetime_mock() insert_datetime_mock()
module.create_archive( module.create_archive(
exclude_patterns=None,
verbosity=None, verbosity=None,
storage_config={'umask': 740},
source_directories=['foo', 'bar'],
repository='repo', repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={'umask': 740},
command='borg', command='borg',
) )
@ -208,11 +232,14 @@ def test_create_archive_with_source_directories_glob_expands():
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
module.create_archive( module.create_archive(
exclude_patterns=None,
verbosity=None, verbosity=None,
storage_config={},
source_directories=['foo*'],
repository='repo', repository='repo',
location_config={
'source_directories': ['foo*'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
command='borg', command='borg',
) )
@ -225,11 +252,14 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([]) flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
module.create_archive( module.create_archive(
exclude_patterns=None,
verbosity=None, verbosity=None,
storage_config={},
source_directories=['foo*'],
repository='repo', repository='repo',
location_config={
'source_directories': ['foo*'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
command='borg', command='borg',
) )
@ -242,11 +272,14 @@ def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
module.create_archive( module.create_archive(
exclude_patterns=None,
verbosity=None, verbosity=None,
storage_config={},
source_directories=['foo*'],
repository='repo', repository='repo',
location_config={
'source_directories': ['foo*'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
command='borg', command='borg',
) )