From 9cafc16052af3002714f97836d90e281e7a7fa70 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 3 Jul 2023 00:08:54 -0700 Subject: [PATCH] For "borgmatic borg", pass the repository to Borg via a Borg-supported environment variable (#575). --- NEWS | 7 ++- borgmatic/borg/borg.py | 3 +- docs/how-to/run-arbitrary-borg-commands.md | 69 ++++++++++++---------- tests/unit/borg/test_borg.py | 68 ++++++++++----------- 4 files changed, 78 insertions(+), 69 deletions(-) diff --git a/NEWS b/NEWS index 1af50d8..f1ecea0 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ 1.8.0.dev0 * #575: BREAKING: For the "borgmatic borg" action, instead of implicitly injecting - repository/archive into the resulting Borg command-line, make repository and archive environment - variables available for explicit use in your commands. See the documentation for more - information: https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/ + repository/archive into the resulting Borg command-line, pass repository to Borg via an + environment variable and make archive available for explicit use in your commands. See the + documentation for more information: + https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/ * #719: Fix an error when running "borg key export" through borgmatic. * #720: Fix an error when dumping a MySQL database and the "exclude_nodump" option is set. * When merging two configuration files, error gracefully if the two files do not adhere to the same diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index 107f140..e0a5692 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -8,7 +8,6 @@ from borgmatic.execute import DO_NOT_CAPTURE, execute_command logger = logging.getLogger(__name__) -REPOSITORYLESS_BORG_COMMANDS = {'serve', None} BORG_SUBCOMMANDS_WITH_SUBCOMMANDS = {'key', 'debug'} @@ -64,7 +63,7 @@ def run_arbitrary_borg( extra_environment=dict( (environment.make_environment(storage_config) or {}), **{ - 'REPOSITORY': repository_path, + 'BORG_REPO': repository_path, 'ARCHIVE': archive if archive else '', }, ), diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index e572038..8962b90 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -33,57 +33,65 @@ arguments: New in version 1.5.15 The way you run Borg with borgmatic is via the `borg` action. Here's a simple example: -```bash -borgmatic borg break-lock '$REPOSITORY' -``` - -This runs Borg's `break-lock` command once on each configured borgmatic -repository, passing the repository path in as an environment variable named -`REPOSITORY`. The single quotes are necessary in order to pass in a literal -`$REPOSITORY` string instead of trying to resolve it from borgmatic's shell -where it's not yet set. - -Prior to version 1.8.0borgmatic -provided the repository name implicitly, attempting to inject it into your -Borg arguments in the right place (which didn't always work). So your -command-line in these older versions looked more like: - ```bash borgmatic borg break-lock ``` -You can also specify Borg options for relevant commands. In borgmatic 1.8.0+, -that looks like: +This runs Borg's `break-lock` command once with each configured borgmatic +repository, passing the repository path in as a Borg-supported environment +variable named `BORG_REPO`. (The native `borgmatic break-lock` action should +be preferred though for most uses.) + +You can also specify Borg options for relevant commands. For instance: ```bash -borgmatic borg rlist --short '$REPOSITORY' +borgmatic borg rlist --short ``` This runs Borg's `rlist` command once on each configured borgmatic repository. -However, the native `borgmatic rlist` action should be preferred for most uses. What if you only want to run Borg on a single configured borgmatic repository when you've got several configured? Not a problem. The `--repository` argument lets you specify the repository to use, either by its path or its label: ```bash -borgmatic borg --repository repo.borg break-lock '$REPOSITORY' +borgmatic borg --repository repo.borg break-lock ``` +And if you need to specify where the repository goes in the command because +there are positional arguments after it: + +```bash +borgmatic borg debug dump-manifest :: root +``` + +The `::` is a Borg placeholder that means: Substitute the repository passed in +by environment variable here. + +Prior to version 1.8.0borgmatic +attempted to inject the repository name directly into your Borg arguments in +the right place (which didn't always work). So your command-line in these +older versions didn't support the `::` + + ### Specifying an archive For borg commands that expect an archive name, you have a few approaches. Here's one: ```bash -borgmatic borg --archive latest list '$REPOSITORY::$ARCHIVE' +borgmatic borg --archive latest list '::$ARCHIVE' ``` +The single quotes are necessary in order to pass in a literal `$ARCHIVE` +string instead of trying to resolve it from borgmatic's shell where it's not +yet set. + Or if you don't need borgmatic to resolve an archive name like `latest`, you can just do: ```bash -borgmatic borg list '$REPOSITORY::your-actual-archive-name' +borgmatic borg list ::your-actual-archive-name ``` Prior to version 1.8.0borgmatic @@ -100,11 +108,11 @@ borgmatic borg --archive latest list these will list an archive: ```bash -borgmatic borg --archive latest list --repo '$REPOSITORY' '$ARCHIVE' +borgmatic borg --archive latest list '$ARCHIVE' ``` ```bash -borgmatic borg list --repo '$REPOSITORY' your-actual-archive-name +borgmatic borg list your-actual-archive-name ``` ### Limitations @@ -126,12 +134,13 @@ borgmatic's `borg` action is not without limitations: * Unlike normal borgmatic actions that support JSON, the `borg` action will not disable certain borgmatic logs to avoid interfering with JSON output. * Prior to version 1.8.0 - borgmatic implicitly supplied the repository/archive name to Borg for you - (based on your borgmatic configuration or the - `borgmatic borg --repository`/`--archive` arguments)—which meant you couldn't - specify the repository/archive directly in the Borg command. Also, in these - older versions of borgmatic, the `borg` action didn't work for any Borg - commands like `borg serve` that do not accept a repository/archive name. + borgmatic implicitly injected the repository/archive arguments on the Borg + command-line for you (based on your borgmatic configuration or the + `borgmatic borg --repository`/`--archive` arguments)—which meant you + couldn't specify the repository/archive directly in the Borg command. Also, + in these older versions of borgmatic, the `borg` action didn't work for any + Borg commands like `borg serve` that do not accept a repository/archive + name. * Prior to version 1.7.13 Unlike other borgmatic actions, the `borg` action captured (and logged) all output, so interactive prompts and flags like `--progress` dit not work as expected. diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index f03bb89..2d7e175 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -13,18 +13,18 @@ def test_run_arbitrary_borg_calls_borg_with_flags(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '$REPOSITORY'), + ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], ) @@ -34,11 +34,11 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '--info', '$REPOSITORY'), + ('borg', 'break-lock', '--info', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.INFO) @@ -46,7 +46,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], ) @@ -56,11 +56,11 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '--debug', '--show-rc', '$REPOSITORY'), + ('borg', 'break-lock', '--debug', '--show-rc', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.DEBUG) @@ -68,7 +68,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], ) @@ -81,18 +81,18 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags(): ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '--lock-wait', '5', '$REPOSITORY'), + ('borg', 'break-lock', '--lock-wait', '5', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], ) @@ -102,18 +102,18 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '$REPOSITORY::$ARCHIVE'), + ('borg', 'break-lock', '::$ARCHIVE'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': 'archive'}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': 'archive'}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY::$ARCHIVE'], + options=['break-lock', '::$ARCHIVE'], archive='archive', ) @@ -124,18 +124,18 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg1', 'break-lock', '$REPOSITORY'), + ('borg1', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg1', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], local_path='borg1', ) @@ -148,18 +148,18 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags() ).and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '--remote-path', 'borg1', '$REPOSITORY'), + ('borg', 'break-lock', '--remote-path', 'borg1', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock', '$REPOSITORY'], + options=['break-lock', '::'], remote_path='borg1', ) @@ -170,18 +170,18 @@ def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list', '--progress', '$REPOSITORY'), + ('borg', 'list', '--progress', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['list', '--progress', '$REPOSITORY'], + options=['list', '--progress', '::'], ) @@ -191,18 +191,18 @@ def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', '$REPOSITORY'), + ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['--', 'break-lock', '$REPOSITORY'], + options=['--', 'break-lock', '::'], ) @@ -216,7 +216,7 @@ def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( @@ -233,11 +233,11 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'key', 'export', '--info', '$REPOSITORY'), + ('borg', 'key', 'export', '--info', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.INFO) @@ -245,7 +245,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['key', 'export', '$REPOSITORY'], + options=['key', 'export', '::'], ) @@ -255,11 +255,11 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_fla flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'debug', 'dump-manifest', '--info', '$REPOSITORY', 'path'), + ('borg', 'debug', 'dump-manifest', '--info', '::', 'path'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', shell=True, - extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, + extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.INFO) @@ -267,5 +267,5 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_fla repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['debug', 'dump-manifest', '$REPOSITORY', 'path'], + options=['debug', 'dump-manifest', '::', 'path'], )