Support for backing up to multiple repositories.

NEWS

@@ -5,6 +5,7 @@
  * Dropped Python 2 support. Now Python 3 only.
  * #18: Fix for README mention of sample files not included in package.
  * #22: Sample files for triggering borgmatic from a systemd timer.
+ * Support for backing up to multiple repositories.
  * To free up space, now pruning backups prior to creating a new backup.
  * Enabled test coverage output during tox runs.
  * Added logo.

README.md

@@ -21,8 +21,9 @@ location:
         - /etc
         - /var/log/syslog*
 
-    # Path to local or remote repository.
+    # Paths to local or remote repositories.
-    repository: user@backupserver:sourcehostname.borg
+    repositories:
+        - user@backupserver:sourcehostname.borg
 
     # Any paths matching these patterns are excluded from backups.
     exclude_patterns:

borgmatic/borg.py

@@ -39,8 +39,7 @@ def _write_exclude_file(exclude_patterns=None):
 
 
 def create_archive(
-    verbosity, storage_config, source_directories, repository, exclude_patterns=None,
+    verbosity, repository, location_config, storage_config, command=COMMAND,
-    command=COMMAND, one_file_system=None, remote_path=None,
 ):
     '''
     Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
@@ -49,17 +48,18 @@ def create_archive(
     sources = tuple(
         itertools.chain.from_iterable(
             glob.glob(directory) or [directory]
-            for directory in source_directories
+            for directory in location_config['source_directories']
         )
     )
 
-    exclude_file = _write_exclude_file(exclude_patterns)
+    exclude_file = _write_exclude_file(location_config.get('exclude_patterns'))
     exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
     umask = storage_config.get('umask', None)
     umask_flags = ('--umask', str(umask)) if umask else ()
-    one_file_system_flags = ('--one-file-system',) if one_file_system else ()
+    one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
+    remote_path = location_config.get('remote_path')
     remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--info', '--stats',),

borgmatic/commands/borgmatic.py

@@ -44,17 +44,23 @@ def main():  # pragma: no cover
         args = parse_arguments(*sys.argv[1:])
         convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename)
         config = validate.parse_configuration(args.config_filename, validate.schema_filename())
-        repository = config['location']['repository']
+        (location, storage, retention, consistency) = (
-        remote_path = config['location']['remote_path']
-        (storage, retention, consistency) = (
             config.get(section_name, {})
-            for section_name in ('storage', 'retention', 'consistency')
+            for section_name in ('location', 'storage', 'retention', 'consistency')
         )
+        remote_path = location.get('remote_path')
 
         borg.initialize(storage)
-        borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
+
-        borg.create_archive(args.verbosity, storage, **config['location'])
+        for repository in location['repositories']:
-        borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
+            borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
+            borg.create_archive(
+                args.verbosity,
+                repository,
+                location,
+                storage,
+            )
+            borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

borgmatic/config/convert.py

@@ -32,9 +32,12 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
         for section_name, section_config in source_config._asdict().items()
     ])
 
-    # Split space-seperated values into actual lists, and merge in excludes.
+    # Split space-seperated values into actual lists, make "repository" into a list, and merge in
-    destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
+    # excludes.
-    destination_config['location']['exclude_patterns'] = source_excludes
+    location = destination_config['location']
+    location['source_directories'] = source_config.location['source_directories'].split(' ')
+    location['repositories'] = [location.pop('repository')]
+    location['exclude_patterns'] = source_excludes
 
     if source_config.consistency['checks']:
         destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')

borgmatic/config/schema.yaml

@@ -25,11 +25,15 @@ map:
                 type: scalar
                 desc: Alternate Borg remote executable. Defaults to "borg".
                 example: borg1
-            repository:
+            repositories:
                 required: True
-                type: scalar
+                seq:
-                desc: Path to local or remote repository (required).
+                    - type: scalar
-                example: user@backupserver:sourcehostname.borg
+                desc: |
+                    Paths to local or remote repositories (required). Multiple repositories are
+                    backed up to in sequence.
+                example:
+                    - user@backupserver:sourcehostname.borg
             exclude_patterns:
                 seq:
                     - type: scalar

borgmatic/tests/integration/config/test_validate.py

@@ -35,7 +35,8 @@ def test_parse_configuration_transforms_file_into_mapping():
                 - /home
                 - /etc
 
-            repository: hostname.borg
+            repositories:
+                - hostname.borg
 
         retention:
             keep_daily: 7
@@ -50,7 +51,7 @@ def test_parse_configuration_transforms_file_into_mapping():
     result = module.parse_configuration('config.yaml', 'schema.yaml')
 
     assert result == {
-        'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
+        'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']},
         'retention': {'keep_daily': 7},
         'consistency': {'checks': ['repository', 'archives']},
     }
@@ -65,7 +66,8 @@ def test_parse_configuration_passes_through_quoted_punctuation():
             source_directories:
                 - /home
 
-            repository: "{}.borg"
+            repositories:
+                - "{}.borg"
         '''.format(escaped_punctuation)
     )
 
@@ -74,7 +76,7 @@ def test_parse_configuration_passes_through_quoted_punctuation():
     assert result == {
         'location': {
             'source_directories': ['/home'],
-            'repository': '{}.borg'.format(string.punctuation),
+            'repositories': ['{}.borg'.format(string.punctuation)],
         },
     }
 
@@ -105,7 +107,8 @@ def test_parse_configuration_raises_for_validation_error():
         '''
         location:
             source_directories: yes
-            repository: hostname.borg
+            repositories:
+                - hostname.borg
         '''
     )
 

borgmatic/tests/unit/config/test_convert.py

@@ -28,7 +28,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
             'location',
             OrderedDict([
                 ('source_directories', ['/home']),
-                ('repository', 'hostname.borg'),
+                ('repositories', ['hostname.borg']),
                 ('exclude_patterns', ['/var']),
             ]),
         ),
@@ -41,7 +41,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
 def test_convert_legacy_parsed_config_splits_space_separated_values():
     flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
     source_config = Parsed_config(
-        location=OrderedDict([('source_directories', '/home /etc')]),
+        location=OrderedDict([('source_directories', '/home /etc'), ('repository', 'hostname.borg')]),
         storage=OrderedDict(),
         retention=OrderedDict(),
         consistency=OrderedDict([('checks', 'repository archives')]),
@@ -56,6 +56,7 @@ def test_convert_legacy_parsed_config_splits_space_separated_values():
             'location',
             OrderedDict([
                 ('source_directories', ['/home', '/etc']),
+                ('repositories', ['hostname.borg']),
                 ('exclude_patterns', ['/var']),
             ]),
         ),

borgmatic/tests/unit/test_borg.py

@@ -77,11 +77,14 @@ def test_create_archive_should_call_borg_with_parameters():
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -93,11 +96,14 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=['exclude'],
         verbosity=None,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': ['exclude'],
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -109,11 +115,14 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=VERBOSITY_SOME,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -125,11 +134,14 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=VERBOSITY_LOTS,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -141,11 +153,14 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={'compression': 'rle'},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'compression': 'rle'},
         command='borg',
     )
 
@@ -157,13 +172,16 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'one_file_system': True,
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
-        one_file_system=True,
     )
 
 
@@ -174,13 +192,16 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'remote_path': 'borg1',
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
-        remote_path='borg1',
     )
 
 
@@ -191,11 +212,14 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
     insert_datetime_mock()
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={'umask': 740},
-        source_directories=['foo', 'bar'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'umask': 740},
         command='borg',
     )
 
@@ -208,11 +232,14 @@ def test_create_archive_with_source_directories_glob_expands():
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo*'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -225,11 +252,14 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo*'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )
 
@@ -242,11 +272,14 @@ def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
     module.create_archive(
-        exclude_patterns=None,
         verbosity=None,
-        storage_config={},
-        source_directories=['foo*'],
         repository='repo',
+        location_config={
+            'source_directories': ['foo*'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
         command='borg',
     )