diff --git a/NEWS b/NEWS index 41ee5fe..978d04f 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,8 @@ -1.7.16.dev0 +1.8.0.dev0 + * #575: BREAKING: For the "borgmatic borg" action, instead of implicitly injecting + repository/archive into the resulting Borg command-line, make repository and archive environment + variables available for explicit use in your commands. See the documentation for more + information: https://torsion.org/borgmatic/docs/how-to/run-arbitrary-borg-commands/ * #719: Fix an error when running "borg key export" through borgmatic. 1.7.15 diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index f815dfd..c5adf89 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -10,7 +10,6 @@ logger = logging.getLogger(__name__) REPOSITORYLESS_BORG_COMMANDS = {'serve', None} BORG_SUBCOMMANDS_WITH_SUBCOMMANDS = {'key', 'debug'} -BORG_SUBCOMMANDS_WITHOUT_REPOSITORY = (('debug', 'info'), ('debug', 'convert-profile'), ()) def run_arbitrary_borg( @@ -25,7 +24,8 @@ def run_arbitrary_borg( ''' Given a local or remote repository path, a storage config dict, the local Borg version, a sequence of arbitrary command-line Borg options, and an optional archive name, run an arbitrary - Borg command on the given repository/archive. + Borg command, passing in $REPOSITORY and $ARCHIVE environment variables for optional use in the + commmand. ''' borgmatic.logger.add_custom_log_levels() lock_wait = storage_config.get('lock_wait', None) @@ -46,29 +46,26 @@ def run_arbitrary_borg( borg_command = () command_options = () - if borg_command in BORG_SUBCOMMANDS_WITHOUT_REPOSITORY: - repository_archive_flags = () - elif archive: - repository_archive_flags = flags.make_repository_archive_flags( - repository_path, archive, local_borg_version - ) - else: - repository_archive_flags = flags.make_repository_flags(repository_path, local_borg_version) - full_command = ( (local_path,) + borg_command - + repository_archive_flags - + command_options + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + flags.make_flags('remote-path', remote_path) + flags.make_flags('lock-wait', lock_wait) + + command_options ) return execute_command( full_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path, - extra_environment=environment.make_environment(storage_config), + shell=True, + extra_environment=dict( + (environment.make_environment(storage_config) or {}), + **{ + 'REPOSITORY': repository_path, + 'ARCHIVE': archive if archive else '', + }, + ), ) diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index ea265ea..e572038 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -7,7 +7,7 @@ eleventyNavigation: --- ## Running Borg with borgmatic -Borg has several commands (and options) that borgmatic does not currently +Borg has several commands and options that borgmatic does not currently support. Sometimes though, as a borgmatic user, you may find yourself wanting to take advantage of these off-the-beaten-path Borg features. You could of course drop down to running Borg directly. But then you'd give up all the @@ -17,11 +17,11 @@ request](https://torsion.org/borgmatic/#contributing) to add the feature. But what if you need it *now*? That's where borgmatic's support for running "arbitrary" Borg commands comes -in. Running Borg commands with borgmatic takes advantage of the following, all -based on your borgmatic configuration files or command-line arguments: +in. Running these Borg commands with borgmatic can take advantage of the +following, all based on your borgmatic configuration files or command-line +arguments: - * configured repositories (automatically runs your Borg command once for each - one) + * configured repositories, running your Borg command once for each one * local and remote Borg binary paths * SSH settings and Borg environment variables * lock wait settings @@ -33,37 +33,78 @@ based on your borgmatic configuration files or command-line arguments: New in version 1.5.15 The way you run Borg with borgmatic is via the `borg` action. Here's a simple example: +```bash +borgmatic borg break-lock '$REPOSITORY' +``` + +This runs Borg's `break-lock` command once on each configured borgmatic +repository, passing the repository path in as an environment variable named +`REPOSITORY`. The single quotes are necessary in order to pass in a literal +`$REPOSITORY` string instead of trying to resolve it from borgmatic's shell +where it's not yet set. + +Prior to version 1.8.0borgmatic +provided the repository name implicitly, attempting to inject it into your +Borg arguments in the right place (which didn't always work). So your +command-line in these older versions looked more like: + ```bash borgmatic borg break-lock ``` -(No `borg` action in borgmatic? Time to upgrade!) - -This runs Borg's `break-lock` command once on each configured borgmatic -repository. Notice how the repository isn't present in the specified Borg -options, as that part is provided by borgmatic. - -You can also specify Borg options for relevant commands: +You can also specify Borg options for relevant commands. In borgmatic 1.8.0+, +that looks like: ```bash -borgmatic borg rlist --short +borgmatic borg rlist --short '$REPOSITORY' ``` This runs Borg's `rlist` command once on each configured borgmatic repository. -(The native `borgmatic rlist` action should be preferred for most use.) +However, the native `borgmatic rlist` action should be preferred for most uses. What if you only want to run Borg on a single configured borgmatic repository when you've got several configured? Not a problem. The `--repository` argument lets you specify the repository to use, either by its path or its label: ```bash -borgmatic borg --repository repo.borg break-lock +borgmatic borg --repository repo.borg break-lock '$REPOSITORY' ``` -And what about a single archive? +### Specifying an archive + +For borg commands that expect an archive name, you have a few approaches. +Here's one: ```bash -borgmatic borg --archive your-archive-name rlist +borgmatic borg --archive latest list '$REPOSITORY::$ARCHIVE' +``` + +Or if you don't need borgmatic to resolve an archive name like `latest`, you +can just do: + +```bash +borgmatic borg list '$REPOSITORY::your-actual-archive-name' +``` + +Prior to version 1.8.0borgmatic +provided the archive name implicitly along with the repository, attempting to +inject it into your Borg arguments in the right place (which didn't always +work). So your command-line in these older versions of borgmatic looked more +like: + +```bash +borgmatic borg --archive latest list +``` + +With Borg version 2.x Either of +these will list an archive: + +```bash +borgmatic borg --archive latest list --repo '$REPOSITORY' '$ARCHIVE' +``` + +```bash +borgmatic borg list --repo '$REPOSITORY' your-actual-archive-name ``` ### Limitations @@ -71,14 +112,10 @@ borgmatic borg --archive your-archive-name rlist borgmatic's `borg` action is not without limitations: * The Borg command you want to run (`create`, `list`, etc.) *must* come first - after the `borg` action. If you have any other Borg options to specify, - provide them after. For instance, `borgmatic borg list --progress` will work, - but `borgmatic borg --progress list` will not. - * borgmatic supplies the repository/archive name to Borg for you (based on - your borgmatic configuration or the `borgmatic borg --repository`/`--archive` - arguments), so do not specify the repository/archive otherwise. - * The `borg` action will not currently work for any Borg commands like `borg - serve` that do not accept a repository/archive name. + after the `borg` action (and any borgmatic-specific arguments). If you have + other Borg options to specify, provide them after. For instance, + `borgmatic borg list --progress ...` will work, but + `borgmatic borg --progress list ...` will not. * Do not specify any global borgmatic arguments to the right of the `borg` action. (They will be passed to Borg instead of borgmatic.) If you have global borgmatic arguments, specify them *before* the `borg` action. @@ -88,10 +125,17 @@ borgmatic's `borg` action is not without limitations: borgmatic action. In this case, only the Borg command is run. * Unlike normal borgmatic actions that support JSON, the `borg` action will not disable certain borgmatic logs to avoid interfering with JSON output. - * Unlike other borgmatic actions, the `borg` action captures (and logs) all - output, so interactive prompts and flags like `--progress` will not work as - expected. New in version - 1.7.13 borgmatic now runs the `borg` action without capturing output, + * Prior to version 1.8.0 + borgmatic implicitly supplied the repository/archive name to Borg for you + (based on your borgmatic configuration or the + `borgmatic borg --repository`/`--archive` arguments)—which meant you couldn't + specify the repository/archive directly in the Borg command. Also, in these + older versions of borgmatic, the `borg` action didn't work for any Borg + commands like `borg serve` that do not accept a repository/archive name. + * Prior to version 1.7.13 Unlike + other borgmatic actions, the `borg` action captured (and logged) all output, + so interactive prompts and flags like `--progress` dit not work as expected. + In new versions, borgmatic runs the `borg` action without capturing output, so interactive prompts work. In general, this `borgmatic borg` feature should be considered an escape diff --git a/setup.py b/setup.py index 37ff017..d6a01fd 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.16.dev0' +VERSION = '1.8.0.dev0' setup( diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 5ae013f..f03bb89 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -10,35 +10,35 @@ from ..test_verbosity import insert_logging_mock def test_run_arbitrary_borg_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo'), + ('borg', 'break-lock', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], ) def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo', '--info'), + ('borg', 'break-lock', '--info', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.INFO) @@ -46,21 +46,21 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], ) def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo', '--debug', '--show-rc'), + ('borg', 'break-lock', '--debug', '--show-rc', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) insert_logging_mock(logging.DEBUG) @@ -68,7 +68,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], ) @@ -76,46 +76,44 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') ) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo', '--lock-wait', '5'), + ('borg', 'break-lock', '--lock-wait', '5', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], ) def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( - ('repo::archive',) - ) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo::archive'), + ('borg', 'break-lock', '$REPOSITORY::$ARCHIVE'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': 'archive'}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY::$ARCHIVE'], archive='archive', ) @@ -123,21 +121,21 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg1', 'break-lock', 'repo'), + ('borg1', 'break-lock', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg1', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], local_path='borg1', ) @@ -145,23 +143,23 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return( ('--remote-path', 'borg1') ).and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo', '--remote-path', 'borg1'), + ('borg', 'break-lock', '--remote-path', 'borg1', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['break-lock'], + options=['break-lock', '$REPOSITORY'], remote_path='borg1', ) @@ -169,56 +167,56 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags() def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'list', 'repo', '--progress'), + ('borg', 'list', '--progress', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['list', '--progress'], + options=['list', '--progress', '$REPOSITORY'], ) def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'break-lock', 'repo'), + ('borg', 'break-lock', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['--', 'break-lock'], + options=['--', 'break-lock', '$REPOSITORY'], ) def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').never() flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg',), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) module.run_arbitrary_borg( @@ -229,85 +227,45 @@ def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): ) -def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository(): +def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'key', 'export', 'repo'), + ('borg', 'key', 'export', '--info', '$REPOSITORY'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) + insert_logging_mock(logging.INFO) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['key', 'export'], + options=['key', 'export', '$REPOSITORY'], ) -def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository(): +def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( - ('borg', 'debug', 'dump-manifest', 'repo', 'path'), + ('borg', 'debug', 'dump-manifest', '--info', '$REPOSITORY', 'path'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', - extra_environment=None, + shell=True, + extra_environment={'REPOSITORY': 'repo', 'ARCHIVE': ''}, ) + insert_logging_mock(logging.INFO) module.run_arbitrary_borg( repository_path='repo', storage_config={}, local_borg_version='1.2.3', - options=['debug', 'dump-manifest', 'path'], - ) - - -def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repository(): - flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') - flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').never() - flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'debug', 'info'), - output_file=module.borgmatic.execute.DO_NOT_CAPTURE, - borg_local_path='borg', - extra_environment=None, - ) - - module.run_arbitrary_borg( - repository_path='repo', - storage_config={}, - local_borg_version='1.2.3', - options=['debug', 'info'], - ) - - -def test_run_arbitrary_borg_with_debug_convert_profile_command_does_not_pass_borg_repository(): - flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') - flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_repository_flags').never() - flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.environment).should_receive('make_environment') - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'debug', 'convert-profile', 'in', 'out'), - output_file=module.borgmatic.execute.DO_NOT_CAPTURE, - borg_local_path='borg', - extra_environment=None, - ) - - module.run_arbitrary_borg( - repository_path='repo', - storage_config={}, - local_borg_version='1.2.3', - options=['debug', 'convert-profile', 'in', 'out'], + options=['debug', 'dump-manifest', '$REPOSITORY', 'path'], )