Store configs used to create an archive in the archive and add borgmatic bootstrap (#697).

Merge pull request #71 from diivi/feat/store-config-in-archive

borgmatic/actions/config/bootstrap.py

@@ -0,0 +1,84 @@
+import json
+import logging
+import os
+
+import borgmatic.borg.extract
+import borgmatic.borg.rlist
+import borgmatic.config.validate
+import borgmatic.hooks.command
+from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+
+logger = logging.getLogger(__name__)
+
+
+def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version):
+    '''
+    Given:
+    The bootstrap arguments, which include the repository and archive name, borgmatic source directory,
+    destination directory, and whether to strip components.
+    The global arguments, which include the dry run flag
+    and the local borg version,
+    Return:
+    The config paths from the manifest.json file in the borgmatic source directory after extracting it from the
+    repository.
+    '''
+    borgmatic_source_directory = (
+        bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+    )
+    borgmatic_manifest_path = os.path.expanduser(
+        os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
+    )
+    extract_process = borgmatic.borg.extract.extract_archive(
+        global_arguments.dry_run,
+        bootstrap_arguments.repository,
+        borgmatic.borg.rlist.resolve_archive_name(
+            bootstrap_arguments.repository,
+            bootstrap_arguments.archive,
+            {},
+            local_borg_version,
+            global_arguments,
+        ),
+        [borgmatic_manifest_path],
+        {},
+        {},
+        local_borg_version,
+        global_arguments,
+        extract_to_stdout=True,
+    )
+
+    manifest_data = json.loads(extract_process.stdout.read())
+
+    return manifest_data['config_paths']
+
+
+def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
+    '''
+    Run the "bootstrap" action for the given repository.
+    '''
+    manifest_config_paths = get_config_paths(
+        bootstrap_arguments, global_arguments, local_borg_version
+    )
+
+    for config_path in manifest_config_paths:
+        logger.info('Bootstrapping config path %s', config_path)
+
+        borgmatic.borg.extract.extract_archive(
+            global_arguments.dry_run,
+            bootstrap_arguments.repository,
+            borgmatic.borg.rlist.resolve_archive_name(
+                bootstrap_arguments.repository,
+                bootstrap_arguments.archive,
+                {},
+                local_borg_version,
+                global_arguments,
+            ),
+            [config_path],
+            {},
+            {},
+            local_borg_version,
+            global_arguments,
+            extract_to_stdout=False,
+            destination_path=bootstrap_arguments.destination,
+            strip_components=bootstrap_arguments.strip_components,
+            progress=bootstrap_arguments.progress,
+        )

borgmatic/actions/create.py

@@ -1,15 +1,51 @@
 import json
 import logging
+import os
+
+try:
+    import importlib_metadata
+except ModuleNotFoundError:  # pragma: nocover
+    import importlib.metadata as importlib_metadata
 
 import borgmatic.borg.create
 import borgmatic.config.validate
 import borgmatic.hooks.command
 import borgmatic.hooks.dispatch
 import borgmatic.hooks.dump
+from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
 
 logger = logging.getLogger(__name__)
 
 
+def create_borgmatic_manifest(location, config_paths, dry_run):
+    '''
+    Create a borgmatic manifest file to store the paths to the configuration files used to create
+    the archive.
+    '''
+    if dry_run:
+        return
+
+    borgmatic_source_directory = location.get(
+        'borgmatic_source_directory', DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+    )
+
+    borgmatic_manifest_path = os.path.expanduser(
+        os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
+    )
+
+    if not os.path.exists(borgmatic_manifest_path):
+        os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True)
+
+    with open(borgmatic_manifest_path, 'w') as config_list_file:
+        json.dump(
+            {
+                'borgmatic_version': importlib_metadata.version('borgmatic'),
+                'config_paths': config_paths,
+            },
+            config_list_file,
+        )
+
+
 def run_create(
     config_filename,
     repository,
@@ -59,6 +95,9 @@ def run_create(
         location,
         global_arguments.dry_run,
     )
+    create_borgmatic_manifest(
+        location, global_arguments.used_config_paths, global_arguments.dry_run
+    )
     stream_processes = [process for processes in active_dumps.values() for process in processes]
 
     json_output = borgmatic.borg.create.create_archive(

borgmatic/borg/create.py

@@ -351,7 +351,9 @@ def create_archive(
     sources = deduplicate_directories(
         map_directories_to_devices(
             expand_directories(
-                tuple(location_config.get('source_directories', ())) + borgmatic_source_directories
+                tuple(location_config.get('source_directories', ()))
+                + borgmatic_source_directories
+                + tuple(global_arguments.used_config_paths)
             )
         ),
         additional_directory_devices=map_directories_to_devices(

borgmatic/commands/arguments.py

@@ -1,4 +1,6 @@
+import argparse
 import collections
+import itertools
 from argparse import Action, ArgumentParser
 
 from borgmatic.config import collect
@@ -9,6 +11,8 @@ SUBPARSER_ALIASES = {
     'compact': [],
     'create': ['-C'],
     'check': ['-k'],
+    'config': [],
+    'config_bootstrap': [],
     'extract': ['-x'],
     'export-tar': [],
     'mount': ['-m'],
@@ -24,6 +28,27 @@ SUBPARSER_ALIASES = {
 }
 
 
+def get_unparsable_arguments(remaining_subparser_arguments):
+    '''
+    Determine the remaining arguments that no subparsers have consumed.
+    '''
+    if remaining_subparser_arguments:
+        remaining_arguments = [
+            argument
+            for argument in dict.fromkeys(
+                itertools.chain.from_iterable(remaining_subparser_arguments)
+            ).keys()
+            if all(
+                argument in subparser_arguments
+                for subparser_arguments in remaining_subparser_arguments
+            )
+        ]
+    else:
+        remaining_arguments = []
+
+    return remaining_arguments
+
+
 def parse_subparser_arguments(unparsed_arguments, subparsers):
     '''
     Given a sequence of arguments and a dict from subparser name to argparse.ArgumentParser
@@ -40,6 +65,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
         for subparser_name, aliases in SUBPARSER_ALIASES.items()
         for alias in aliases
     }
+    subcommand_parsers_mapping = {
+        'config': ['bootstrap'],
+    }
 
     # If the "borg" action is used, skip all other subparsers. This avoids confusion like
     # "borg list" triggering borgmatic's own list action.
@@ -56,7 +84,9 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
         # If a parsed value happens to be the same as the name of a subparser, remove it from the
         # remaining arguments. This prevents, for instance, "check --only extract" from triggering
         # the "extract" subparser.
-        parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+        parsed, unused_remaining = subparser.parse_known_args(
+            [argument for argument in unparsed_arguments if argument != canonical_name]
+        )
         for value in vars(parsed).values():
             if isinstance(value, str):
                 if value in subparsers:
@@ -66,7 +96,16 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
                     if item in subparsers:
                         remaining_arguments.remove(item)
 
-        arguments[canonical_name] = parsed
+        arguments[canonical_name] = None if canonical_name in subcommand_parsers_mapping else parsed
+
+    for argument in arguments:
+        if not arguments[argument]:
+            if not any(
+                subcommand in arguments for subcommand in subcommand_parsers_mapping[argument]
+            ):
+                raise ValueError(
+                    f'Missing subcommand for {argument}. Expected one of {subcommand_parsers_mapping[argument]}'
+                )
 
     # If no actions are explicitly requested, assume defaults.
     if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
@@ -77,13 +116,22 @@ def parse_subparser_arguments(unparsed_arguments, subparsers):
 
     remaining_arguments = list(unparsed_arguments)
 
-    # Now ask each subparser, one by one, to greedily consume arguments.
+    # Now ask each subparser, one by one, to greedily consume arguments, from last to first. This
-    for subparser_name, subparser in subparsers.items():
+    # allows subparsers to consume arguments before their parent subparsers do.
+    remaining_subparser_arguments = []
+
+    for subparser_name, subparser in reversed(subparsers.items()):
         if subparser_name not in arguments.keys():
             continue
 
         subparser = subparsers[subparser_name]
-        unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+        unused_parsed, remaining = subparser.parse_known_args(
+            [argument for argument in unparsed_arguments if argument != subparser_name]
+        )
+        remaining_subparser_arguments.append(remaining)
+
+    if remaining_subparser_arguments:
+        remaining_arguments = get_unparsable_arguments(remaining_subparser_arguments)
 
     # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the
     # "borg" action.
@@ -109,7 +157,7 @@ class Extend_action(Action):
         items = getattr(namespace, self.dest, None)
 
         if items:
-            items.extend(values)
+            items.extend(values)  # pragma: no cover
         else:
             setattr(namespace, self.dest, list(values))
 
@@ -563,6 +611,71 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
+    config_parser = subparsers.add_parser(
+        'config',
+        aliases=SUBPARSER_ALIASES['config'],
+        help='Perform configuration file related operations',
+        description='Perform configuration file related operations',
+        add_help=False,
+    )
+
+    config_group = config_parser.add_argument_group('config arguments')
+    config_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    config_subparsers = config_parser.add_subparsers(
+        title='config subcommands',
+        description='Valid subcommands for config',
+        help='Additional help',
+    )
+
+    config_bootstrap_parser = config_subparsers.add_parser(
+        'bootstrap',
+        aliases=SUBPARSER_ALIASES['config_bootstrap'],
+        help='Extract the config files used to create a borgmatic repository',
+        description='Extract config files that were used to create a borgmatic repository during the "create" operation',
+        add_help=False,
+    )
+    config_bootstrap_group = config_bootstrap_parser.add_argument_group(
+        'config bootstrap arguments'
+    )
+    config_bootstrap_group.add_argument(
+        '--repository',
+        help='Path of repository to extract config files from',
+        required=True,
+    )
+    config_bootstrap_group.add_argument(
+        '--borgmatic-source-directory',
+        help='Path that stores the config files used to create an archive and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic',
+    )
+    config_bootstrap_group.add_argument(
+        '--archive',
+        help='Name of archive to extract config files from, defaults to "latest"',
+        default='latest',
+    )
+    config_bootstrap_group.add_argument(
+        '--destination',
+        metavar='PATH',
+        dest='destination',
+        help='Directory to extract config files into, defaults to /',
+        default='/',
+    )
+    config_bootstrap_group.add_argument(
+        '--strip-components',
+        type=lambda number: number if number == 'all' else int(number),
+        metavar='NUMBER',
+        help='Number of leading path components to remove from each extracted path or "all" to strip all leading path components. Skip paths with fewer elements',
+    )
+    config_bootstrap_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is extracted',
+    )
+    config_bootstrap_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
     export_tar_parser = subparsers.add_parser(
         'export-tar',
         aliases=SUBPARSER_ALIASES['export-tar'],
@@ -973,7 +1086,28 @@ def make_parsers():
     )
     borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    return top_level_parser, subparsers
+    merged_subparsers = argparse._SubParsersAction(
+        None, None, metavar=None, dest='merged', parser_class=None
+    )
+
+    merged_subparsers = merge_subparsers(subparsers, config_subparsers)
+
+    return top_level_parser, merged_subparsers
+
+
+def merge_subparsers(*subparsers):
+    '''
+    Merge multiple subparsers into a single subparser.
+    '''
+    merged_subparsers = argparse._SubParsersAction(
+        None, None, metavar=None, dest='merged', parser_class=None
+    )
+
+    for subparser in subparsers:
+        for name, subparser in subparser.choices.items():
+            merged_subparsers._name_parser_map[name] = subparser
+
+    return merged_subparsers
 
 
 def parse_arguments(*unparsed_arguments):
@@ -986,6 +1120,16 @@ def parse_arguments(*unparsed_arguments):
     arguments, remaining_arguments = parse_subparser_arguments(
         unparsed_arguments, subparsers.choices
     )
+
+    if (
+        'bootstrap' in arguments.keys()
+        and 'config' in arguments.keys()
+        and len(arguments.keys()) > 2
+    ):
+        raise ValueError(
+            'The bootstrap action cannot be combined with other actions. Please run it separately.'
+        )
+
     arguments['global'] = top_level_parser.parse_args(remaining_arguments)
 
     if arguments['global'].excludes_filename:

borgmatic/commands/borgmatic.py

@@ -18,6 +18,7 @@ import borgmatic.actions.borg
 import borgmatic.actions.break_lock
 import borgmatic.actions.check
 import borgmatic.actions.compact
+import borgmatic.actions.config.bootstrap
 import borgmatic.actions.create
 import borgmatic.actions.export_tar
 import borgmatic.actions.extract
@@ -622,11 +623,37 @@ def collect_configuration_run_summary_logs(configs, arguments):
         if 'extract' in arguments or 'mount' in arguments:
             validate.guard_single_repository_selected(repository, configs)
 
-        validate.guard_configuration_contains_repository(repository, configs)
+        if 'bootstrap' not in arguments:
+            validate.guard_configuration_contains_repository(repository, configs)
     except ValueError as error:
         yield from log_error_records(str(error))
         return
 
+    if 'bootstrap' in arguments:
+        # no configuration file is needed for bootstrap
+        local_borg_version = borg_version.local_borg_version({}, 'borg')
+        try:
+            borgmatic.actions.config.bootstrap.run_bootstrap(
+                arguments['bootstrap'], arguments['global'], local_borg_version
+            )
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.INFO,
+                    levelname='INFO',
+                    msg='Bootstrap successful',
+                )
+            )
+        except (
+            CalledProcessError,
+            ValueError,
+            OSError,
+            json.JSONDecodeError,
+            KeyError,
+        ) as error:
+            yield from log_error_records('Error running bootstrap', error)
+
+        return
+
     if not configs:
         yield from log_error_records(
             f"{' '.join(arguments['global'].config_paths)}: No valid configuration files found",
@@ -733,6 +760,7 @@ def main():  # pragma: no cover
         sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
+    global_arguments.used_config_paths = list(config_filenames)
     configs, parse_logs = load_configurations(
         config_filenames, global_arguments.overrides, global_arguments.resolve_env
     )

tests/integration/commands/test_arguments.py

@@ -1,3 +1,5 @@
+import argparse
+
 import pytest
 from flexmock import flexmock
 
@@ -298,6 +300,13 @@ def test_parse_arguments_disallows_paths_unless_action_consumes_it():
         module.parse_arguments('--config', 'myconfig', '--path', 'test')
 
 
+def test_parse_arguments_disallows_other_actions_with_config_bootstrap():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    with pytest.raises(ValueError):
+        module.parse_arguments('config', 'bootstrap', '--repository', 'test.borg', 'list')
+
+
 def test_parse_arguments_allows_archive_with_extract():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
@@ -523,3 +532,26 @@ def test_parse_arguments_extract_with_check_only_extract_does_not_raise():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
     module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract')
+
+
+def test_merging_two_subparser_collections_merges_their_choices():
+    top_level_parser = argparse.ArgumentParser()
+
+    subparsers = top_level_parser.add_subparsers()
+    subparser1 = subparsers.add_parser('subparser1')
+
+    subparser2 = subparsers.add_parser('subparser2')
+    subsubparsers = subparser2.add_subparsers()
+    subsubparser1 = subsubparsers.add_parser('subsubparser1')
+
+    merged_subparsers = argparse._SubParsersAction(
+        None, None, metavar=None, dest='merged', parser_class=None
+    )
+
+    merged_subparsers = module.merge_subparsers(subparsers, subsubparsers)
+
+    assert merged_subparsers.choices == {
+        'subparser1': subparser1,
+        'subparser2': subparser2,
+        'subsubparser1': subsubparser1,
+    }

tests/unit/actions/config/test_bootstrap.py

@@ -0,0 +1,56 @@
+from flexmock import flexmock
+
+from borgmatic.actions.config import bootstrap as module
+
+
+def test_get_config_paths_returns_list_of_config_paths():
+    bootstrap_arguments = flexmock(
+        borgmatic_source_directory=None,
+        repository='repo',
+        archive='archive',
+    )
+    global_arguments = flexmock(
+        dry_run=False,
+    )
+    local_borg_version = flexmock()
+    extract_process = flexmock(
+        stdout=flexmock(
+            read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}',
+        ),
+    )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
+        extract_process
+    )
+    flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
+        'archive'
+    )
+    assert module.get_config_paths(bootstrap_arguments, global_arguments, local_borg_version) == [
+        '/borgmatic/config.yaml'
+    ]
+
+
+def test_run_bootstrap_does_not_raise():
+    bootstrap_arguments = flexmock(
+        repository='repo',
+        archive='archive',
+        destination='dest',
+        strip_components=1,
+        progress=False,
+        borgmatic_source_directory='/borgmatic',
+    )
+    global_arguments = flexmock(
+        dry_run=False,
+    )
+    local_borg_version = flexmock()
+    extract_process = flexmock(
+        stdout=flexmock(
+            read=lambda: '{"config_paths": ["/borgmatic/config.yaml"]}',
+        ),
+    )
+    flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').and_return(
+        extract_process
+    ).twice()
+    flexmock(module.borgmatic.borg.rlist).should_receive('resolve_archive_name').and_return(
+        'archive'
+    )
+    module.run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version)

tests/unit/actions/test_create.py

@@ -1,3 +1,5 @@
+import sys
+
 from flexmock import flexmock
 
 from borgmatic.actions import create as module
@@ -7,6 +9,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
     flexmock(module.logger).answer = lambda message: None
     flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never()
     flexmock(module.borgmatic.borg.create).should_receive('create_archive').once()
+    flexmock(module).should_receive('create_borgmatic_manifest').once()
     flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2)
     flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({})
     flexmock(module.borgmatic.hooks.dispatch).should_receive(
@@ -19,7 +22,7 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository():
         json=flexmock(),
         list_files=flexmock(),
     )
-    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
+    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[])
 
     list(
         module.run_create(
@@ -45,6 +48,7 @@ def test_run_create_runs_with_selected_repository():
         'repositories_match'
     ).once().and_return(True)
     flexmock(module.borgmatic.borg.create).should_receive('create_archive').once()
+    flexmock(module).should_receive('create_borgmatic_manifest').once()
     create_arguments = flexmock(
         repository=flexmock(),
         progress=flexmock(),
@@ -52,7 +56,7 @@ def test_run_create_runs_with_selected_repository():
         json=flexmock(),
         list_files=flexmock(),
     )
-    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
+    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[])
 
     list(
         module.run_create(
@@ -78,6 +82,7 @@ def test_run_create_bails_if_repository_does_not_match():
         'repositories_match'
     ).once().and_return(False)
     flexmock(module.borgmatic.borg.create).should_receive('create_archive').never()
+    flexmock(module).should_receive('create_borgmatic_manifest').never()
     create_arguments = flexmock(
         repository=flexmock(),
         progress=flexmock(),
@@ -85,7 +90,7 @@ def test_run_create_bails_if_repository_does_not_match():
         json=flexmock(),
         list_files=flexmock(),
     )
-    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False)
+    global_arguments = flexmock(monitoring_verbosity=1, dry_run=False, used_config_paths=[])
 
     list(
         module.run_create(
@@ -103,3 +108,42 @@ def test_run_create_bails_if_repository_does_not_match():
             remote_path=None,
         )
     )
+
+
+def test_create_borgmatic_manifest_creates_manifest_file():
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.os).should_receive('makedirs').and_return(True)
+
+    flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0')
+    flexmock(module.json).should_receive('dump').and_return(True)
+
+    module.create_borgmatic_manifest({}, 'test.yaml', False)
+
+
+def test_create_borgmatic_manifest_creates_manifest_file_with_custom_borgmatic_source_directory():
+    flexmock(module.os.path).should_receive('join').with_args(
+        '/borgmatic', 'bootstrap', 'manifest.json'
+    ).and_return('/borgmatic/bootstrap/manifest.json')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.os).should_receive('makedirs').and_return(True)
+
+    flexmock(module.importlib_metadata).should_receive('version').and_return('1.0.0')
+    flexmock(sys.modules['builtins']).should_receive('open').with_args(
+        '/borgmatic/bootstrap/manifest.json', 'w'
+    ).and_return(
+        flexmock(
+            __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None),
+            __exit__=lambda *args: None,
+        )
+    )
+    flexmock(module.json).should_receive('dump').and_return(True)
+
+    module.create_borgmatic_manifest(
+        {'borgmatic_source_directory': '/borgmatic'}, 'test.yaml', False
+    )
+
+
+def test_create_borgmatic_manifest_does_not_create_manifest_file_on_dry_run():
+    flexmock(module.os.path).should_receive('expanduser').never()
+
+    module.create_borgmatic_manifest({}, 'test.yaml', True)

tests/unit/borg/test_create.py

@@ -492,7 +492,7 @@ def test_create_archive_calls_borg_with_parameters():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -536,7 +536,7 @@ def test_create_archive_calls_borg_with_environment():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -582,7 +582,56 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
+    )
+
+
+def test_create_archive_with_sources_and_used_config_paths_calls_borg_with_sources_and_config_paths():
+    flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels')
+    flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER
+    flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('deduplicate_directories').and_return(
+        ('foo', 'bar', '/etc/borgmatic/config.yaml')
+    )
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
+    flexmock(module).should_receive('expand_directories').with_args([]).and_return(())
+    flexmock(module).should_receive('expand_directories').with_args(
+        ('foo', 'bar', '/etc/borgmatic/config.yaml')
+    ).and_return(('foo', 'bar', '/etc/borgmatic/config.yaml'))
+    flexmock(module).should_receive('expand_directories').with_args([]).and_return(())
+    flexmock(module).should_receive('pattern_root_directories').and_return([])
+    flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
+    flexmock(module).should_receive('expand_home_directories').and_return(())
+    flexmock(module).should_receive('write_pattern_file').and_return(None)
+    flexmock(module).should_receive('make_list_filter_flags').and_return('FOO')
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module).should_receive('ensure_files_readable')
+    flexmock(module).should_receive('make_pattern_flags').and_return(())
+    flexmock(module).should_receive('make_exclude_flags').and_return(())
+    flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
+        (f'repo::{DEFAULT_ARCHIVE_NAME}',)
+    )
+    environment = {'BORG_THINGY': 'YUP'}
+    flexmock(module.environment).should_receive('make_environment').and_return(environment)
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('/etc/borgmatic/config.yaml',),
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+        working_directory=None,
+        extra_environment=environment,
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository_path='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+        },
+        storage_config={},
+        local_borg_version='1.2.3',
+        global_arguments=flexmock(log_json=False, used_config_paths=['/etc/borgmatic/config.yaml']),
     )
 
 
@@ -628,7 +677,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -672,7 +721,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -713,7 +762,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         json=True,
     )
 
@@ -758,7 +807,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -799,7 +848,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         json=True,
     )
 
@@ -843,7 +892,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -889,7 +938,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stats=True,
     )
 
@@ -933,7 +982,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
         },
         storage_config={'checkpoint_interval': 600},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -976,7 +1025,7 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume
         },
         storage_config={'checkpoint_volume': 1024},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1019,7 +1068,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
         },
         storage_config={'chunker_params': '1,2,3,4'},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1062,7 +1111,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
         },
         storage_config={'compression': 'rle'},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1111,7 +1160,7 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_
         },
         storage_config={'upload_rate_limit': 100},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1157,7 +1206,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1201,7 +1250,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1251,7 +1300,7 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter(
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1305,7 +1354,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1361,7 +1410,7 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1416,7 +1465,7 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1471,7 +1520,7 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1515,7 +1564,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1558,7 +1607,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         local_path='borg1',
     )
 
@@ -1602,7 +1651,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         remote_path='borg1',
     )
 
@@ -1646,7 +1695,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
         },
         storage_config={'umask': 740},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1689,7 +1738,7 @@ def test_create_archive_with_log_json_calls_borg_with_log_json_parameters():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=True),
+        global_arguments=flexmock(log_json=True, used_config_paths=[]),
     )
 
 
@@ -1732,7 +1781,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
         },
         storage_config={'lock_wait': 5},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -1775,7 +1824,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stats=True,
     )
 
@@ -1819,7 +1868,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         list_files=True,
     )
 
@@ -1869,7 +1918,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         progress=True,
     )
 
@@ -1913,7 +1962,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         progress=True,
     )
 
@@ -1974,7 +2023,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         progress=True,
         stream_processes=processes,
     )
@@ -2039,7 +2088,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stream_processes=processes,
     )
 
@@ -2107,7 +2156,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stream_processes=processes,
     )
 
@@ -2172,7 +2221,7 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stream_processes=processes,
     )
 
@@ -2213,7 +2262,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         json=True,
     )
 
@@ -2256,7 +2305,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         json=True,
         stats=True,
     )
@@ -2304,7 +2353,7 @@ def test_create_archive_with_source_directories_glob_expands():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2348,7 +2397,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2391,7 +2440,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2434,7 +2483,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
         },
         storage_config={'archive_name_format': 'ARCHIVE_NAME'},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2478,7 +2527,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
         },
         storage_config={'archive_name_format': 'Documents_{hostname}-{now}'},  # noqa: FS003
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2522,7 +2571,7 @@ def test_create_archive_with_repository_accepts_borg_placeholders():
         },
         storage_config={'archive_name_format': 'Documents_{hostname}-{now}'},  # noqa: FS003
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2565,7 +2614,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
         },
         storage_config={'extra_borg_options': {'create': '--extra --options'}},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
     )
 
 
@@ -2626,7 +2675,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read
         },
         storage_config={},
         local_borg_version='1.2.3',
-        global_arguments=flexmock(log_json=False),
+        global_arguments=flexmock(log_json=False, used_config_paths=[]),
         stream_processes=processes,
     )
 
@@ -2652,7 +2701,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_
             },
             storage_config={},
             local_borg_version='1.2.3',
-            global_arguments=flexmock(log_json=False),
+            global_arguments=flexmock(log_json=False, used_config_paths=[]),
         )
 
 

tests/unit/commands/test_arguments.py

@@ -1,5 +1,6 @@
 import collections
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.commands import arguments as module
@@ -164,3 +165,45 @@ def test_parse_subparser_arguments_parses_borg_options_and_skips_other_subparser
     assert arguments == {'borg': action_namespace}
     assert arguments['borg'].options == ['list']
     assert remaining_arguments == []
+
+
+def test_parse_subparser_arguments_raises_error_when_no_subparser_is_specified():
+    action_namespace = flexmock(options=[])
+    subparsers = {
+        'config': flexmock(parse_known_args=lambda arguments: (action_namespace, ['config'])),
+    }
+
+    with pytest.raises(ValueError):
+        module.parse_subparser_arguments(('config',), subparsers)
+
+
+@pytest.mark.parametrize(
+    'arguments, expected',
+    [
+        (
+            (
+                ('--latest', 'archive', 'prune', 'extract', 'list', '--test-flag'),
+                ('--latest', 'archive', 'check', 'extract', 'list', '--test-flag'),
+                ('prune', 'check', 'list', '--test-flag'),
+                ('prune', 'check', 'extract', '--test-flag'),
+            ),
+            [
+                '--test-flag',
+            ],
+        ),
+        (
+            (
+                ('--latest', 'archive', 'prune', 'extract', 'list'),
+                ('--latest', 'archive', 'check', 'extract', 'list'),
+                ('prune', 'check', 'list'),
+                ('prune', 'check', 'extract'),
+            ),
+            [],
+        ),
+        ((), []),
+    ],
+)
+def test_get_unparsable_arguments_returns_remaining_arguments_that_no_subparser_can_parse(
+    arguments, expected
+):
+    assert module.get_unparsable_arguments(arguments) == expected

tests/unit/commands/test_borgmatic.py

@@ -1000,6 +1000,41 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
     assert {log.levelno for log in logs} == {logging.INFO}
 
 
+def test_collect_configuration_run_summary_logs_info_for_success_with_bootstrap():
+    flexmock(module.validate).should_receive('guard_single_repository_selected').never()
+    flexmock(module.validate).should_receive('guard_configuration_contains_repository').never()
+    flexmock(module).should_receive('run_configuration').never()
+    flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap')
+    arguments = {
+        'bootstrap': flexmock(repository='repo'),
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+    }
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
+    assert {log.levelno for log in logs} == {logging.INFO}
+
+
+def test_collect_configuration_run_summary_logs_error_on_bootstrap_failure():
+    flexmock(module.validate).should_receive('guard_single_repository_selected').never()
+    flexmock(module.validate).should_receive('guard_configuration_contains_repository').never()
+    flexmock(module).should_receive('run_configuration').never()
+    flexmock(module.borgmatic.actions.config.bootstrap).should_receive('run_bootstrap').and_raise(
+        ValueError
+    )
+    arguments = {
+        'bootstrap': flexmock(repository='repo'),
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+    }
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
+
+    assert {log.levelno for log in logs} == {logging.CRITICAL}
+
+
 def test_collect_configuration_run_summary_logs_extract_with_repository_error():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
         ValueError