Merge excludes into config file format.

borgmatic/commands/convert_config.py

@@ -11,7 +11,6 @@ from borgmatic.config import convert, generate, legacy, validate
 
 
 DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
-# TODO: Fold excludes into the YAML config file.
 DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
 DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
 
@@ -21,16 +20,27 @@ def parse_arguments(*arguments):
     Given command-line arguments with which this script was invoked, parse the arguments and return
     them as an ArgumentParser instance.
     '''
-    parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.')
+    parser = ArgumentParser(
+        description='''
+            Convert legacy INI-style borgmatic configuration and excludes files to a single YAML
+            configuration file. Note that this replaces any comments from the source files.
+        '''
+    )
     parser.add_argument(
-        '-s', '--source',
+        '-s', '--source-config',
-        dest='source_filename',
+        dest='source_config_filename',
         default=DEFAULT_SOURCE_CONFIG_FILENAME,
         help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
     )
     parser.add_argument(
-        '-d', '--destination',
+        '-e', '--source-excludes',
-        dest='destination_filename',
+        dest='source_excludes_filename',
+        default=DEFAULT_SOURCE_EXCLUDES_FILENAME if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME) else None,
+        help='Excludes filename',
+    )
+    parser.add_argument(
+        '-d', '--destination-config',
+        dest='destination_config_filename',
         default=DEFAULT_DESTINATION_CONFIG_FILENAME,
         help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
     )
@@ -41,12 +51,17 @@ def parse_arguments(*arguments):
 def main():  # pragma: no cover
     try:
         args = parse_arguments(*sys.argv[1:])
-        source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT)
         schema = yaml.round_trip_load(open(validate.schema_filename()).read())
+        source_config = legacy.parse_configuration(args.source_config_filename, legacy.CONFIG_FORMAT)
+        source_excludes = (
+            open(args.source_excludes_filename).read().splitlines()
+            if args.source_excludes_filename
+            else []
+        )
 
-        destination_config = convert.convert_legacy_parsed_config(source_config, schema)
+        destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema)
 
-        generate.write_configuration(args.destination_filename, destination_config)
+        generate.write_configuration(args.destination_config_filename, destination_config)
 
         # TODO: As a backstop, check that the written config can actually be read and parsed, and
         # that it matches the destination config data structure that was written.

borgmatic/config/convert.py

@@ -12,16 +12,15 @@ def _convert_section(source_section_config, section_schema):
     returned CommentedMap.
     '''
     destination_section_config = yaml.comments.CommentedMap(source_section_config)
-    generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT)
 
     return destination_section_config
 
 
-def convert_legacy_parsed_config(source_config, schema):
+def convert_legacy_parsed_config(source_config, source_excludes, schema):
     '''
-    Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its
+    Given a legacy Parsed_config instance loaded from an INI-style config file and a list of exclude
-    corresponding yaml.comments.CommentedMap representation in preparation for actual serialization
+    patterns, convert them to a corresponding yaml.comments.CommentedMap representation in
-    to YAML.
+    preparation for serialization to a single YAML config file.
 
     Additionally, use the given schema as a source of helpful comments to include within the
     returned CommentedMap.
@@ -31,11 +30,21 @@ def convert_legacy_parsed_config(source_config, schema):
         for section_name, section_config in source_config._asdict().items()
     ])
 
+    # Split space-seperated values into actual lists, and merge in excludes.
     destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
+    destination_config['location']['exclude_patterns'] = source_excludes
 
     if source_config.consistency['checks']:
         destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
 
+    # Add comments to each section, and then add comments to the fields in each section.
     generate.add_comments_to_configuration(destination_config, schema)
 
+    for section_name, section_config in destination_config.items():
+        generate.add_comments_to_configuration(
+            section_config,
+            schema['map'][section_name],
+            indent=generate.INDENT,
+        )
+
     return destination_config

borgmatic/config/schema.yaml

@@ -30,6 +30,18 @@ map:
                 type: scalar
                 desc: Path to local or remote repository.
                 example: user@backupserver:sourcehostname.borg
+            exclude_patterns:
+                seq:
+                    - type: scalar
+                desc: |
+                    Exclude patterns. Any paths matching these patterns are excluded from backups.
+                    Globs are expanded. See
+                    https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-patterns for
+                    details.
+                example:
+                    - '*.pyc'
+                    - /home/*/.cache
+                    - /etc/ssl
     storage:
         desc: |
             Repository storage options. See

borgmatic/tests/integration/commands/test_borgmatic.py

@@ -1,5 +1,4 @@
 import os
-import sys
 
 from flexmock import flexmock
 import pytest
@@ -14,7 +13,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults():
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
     assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
-    assert parser.verbosity == None
+    assert parser.verbosity is None
 
 
 def test_parse_arguments_with_filename_arguments_overrides_defaults():
@@ -24,7 +23,7 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
 
     assert parser.config_filename == 'myconfig'
     assert parser.excludes_filename == 'myexcludes'
-    assert parser.verbosity == None
+    assert parser.verbosity is None
 
 
 def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
@@ -33,8 +32,8 @@ def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_non
     parser = module.parse_arguments()
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
-    assert parser.excludes_filename == None
+    assert parser.excludes_filename is None
-    assert parser.verbosity == None
+    assert parser.verbosity is None
 
 
 def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename():
@@ -44,7 +43,7 @@ def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename(
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
     assert parser.excludes_filename == 'myexcludes'
-    assert parser.verbosity == None
+    assert parser.verbosity is None
 
 
 def test_parse_arguments_with_verbosity_flag_overrides_default():
@@ -59,11 +58,6 @@ def test_parse_arguments_with_verbosity_flag_overrides_default():
 
 def test_parse_arguments_with_invalid_arguments_exits():
     flexmock(os.path).should_receive('exists').and_return(True)
-    original_stderr = sys.stderr
-    sys.stderr = sys.stdout
 
-    try:
     with pytest.raises(SystemExit):
         module.parse_arguments('--posix-me-harder')
-    finally:
-        sys.stderr = original_stderr

borgmatic/tests/integration/commands/test_convert_config.py

@@ -1,14 +1,47 @@
+import os
+
+from flexmock import flexmock
+import pytest
+
 from borgmatic.commands import convert_config as module
 
 
 def test_parse_arguments_with_no_arguments_uses_defaults():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
     parser = module.parse_arguments()
 
-    assert parser.source_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
+    assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
-    assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
+    assert parser.source_excludes_filename == module.DEFAULT_SOURCE_EXCLUDES_FILENAME
+    assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
+
 
 def test_parse_arguments_with_filename_arguments_overrides_defaults():
-    parser = module.parse_arguments('--source', 'config', '--destination', 'config.yaml')
+    flexmock(os.path).should_receive('exists').and_return(True)
 
-    assert parser.source_filename == 'config'
+    parser = module.parse_arguments(
-    assert parser.destination_filename == 'config.yaml'
+        '--source-config', 'config',
+        '--source-excludes', 'excludes',
+        '--destination-config', 'config.yaml',
+    )
+
+    assert parser.source_config_filename == 'config'
+    assert parser.source_excludes_filename == 'excludes'
+    assert parser.destination_config_filename == 'config.yaml'
+
+
+def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
+    flexmock(os.path).should_receive('exists').and_return(False)
+
+    parser = module.parse_arguments()
+
+    assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
+    assert parser.source_excludes_filename is None
+    assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
+
+
+def test_parse_arguments_with_invalid_arguments_exits():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
+    with pytest.raises(SystemExit):
+        module.parse_arguments('--posix-me-harder')