Add new "transfer" action for Borg 2 (#557).

NEWS

@@ -1,11 +1,14 @@
 1.7.0.dev0
  * #557: Support for Borg 2 while still working with Borg 1. This includes new borgmatic actions
-   like "rcreate" (replaces "init"), "rlist" (list archives in repository), and "rinfo" (show
+   like "rcreate" (replaces "init"), "rlist" (list archives in repository), "rinfo" (show repository
-   repository info). For the most part, borgmatic tries to smooth over differences between Borg 1
+   info), and "transfer" (for upgrading Borg repositories). For the most part, borgmatic tries to
-   and 2 to make your upgrade process easier. However, there are still a few cases where Borg made
+   smooth over differences between Borg 1 and 2 to make your upgrade process easier. However, there
-   breaking changes. See the Borg 2.0 changelog for more information
+   are still a few cases where Borg made breaking changes. See the Borg 2.0 changelog for more
-   (https://www.borgbackup.org/releases/borg-2.0.html). If you install Borg 2, you'll need to
+   information: https://www.borgbackup.org/releases/borg-2.0.html
-   manually "borg transfer" or "borgmatic transfer" your existing Borg 1 repositories before use.
+ * #557: If you install Borg 2, you'll need to manually upgrade your existing Borg 1 repositories
+   before use. Note that Borg 2 stable is not yet released as of this borgmatic release, so don't
+   use Borg 2 for production until it is! See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-borg
  * #557: Rename several configuration options to match Borg 2: "remote_rate_limit" is now
    "upload_rate_limit", "numeric_owner" is "numeric_ids", and "bsd_flags" is "flags". borgmatic
    still works with the old options.
@@ -14,6 +17,7 @@
    Borg 2.
  * #557: Omitting the "--archive" flag on the "list" action is deprecated when using Borg 2. Use
    the new "rlist" action instead.
+ * #557: The "--dry-run" flag can now be used with the "rcreate"/"init" action.
  * #565: Fix handling of "repository" and "data" consistency checks to prevent invalid Borg flags.
  * #566: Modify "mount" and "extract" actions to require the "--repository" flag when multiple
    repositories are configured.

borgmatic/borg/compact.py

@@ -39,10 +39,13 @@ def compact_segments(
         + flags.make_repository_flags(repository, local_borg_version)
     )
 
-    if not dry_run:
+    if dry_run:
-        execute_command(
+        logging.info(f'{repository}: Skipping compact (dry run)')
-            full_command,
+        return
-            output_log_level=logging.INFO,
+
-            borg_local_path=local_path,
+    execute_command(
-            extra_environment=environment.make_environment(storage_config),
+        full_command,
-        )
+        output_log_level=logging.INFO,
+        borg_local_path=local_path,
+        extra_environment=environment.make_environment(storage_config),
+    )

borgmatic/borg/rcreate.py

@@ -12,11 +12,12 @@ RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
 
 
 def create_repository(
+    dry_run,
     repository,
     storage_config,
     local_borg_version,
     encryption_mode,
-    key_repository=None,
+    source_repository=None,
     copy_crypt_key=False,
     append_only=None,
     storage_quota=None,
@@ -25,10 +26,10 @@ def create_repository(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage configuration dict, the local Borg version, a
+    Given a dry-run flag, a local or remote repository path, a storage configuration dict, the local
-    Borg encryption mode, the path to another repo whose key material should be reused, whether the
+    Borg version, a Borg encryption mode, the path to another repo whose key material should be
-    repository should be append-only, and the storage quota to use, create the repository. If the
+    reused, whether the repository should be append-only, and the storage quota to use, create the
-    repository already exists, then log and skip creation.
+    repository. If the repository already exists, then log and skip creation.
     '''
     try:
         rinfo.display_repository_info(
@@ -39,7 +40,7 @@ def create_repository(
             local_path,
             remote_path,
         )
-        logger.info('Repository already exists. Skipping creation.')
+        logger.info(f'{repository}: Repository already exists. Skipping creation.')
         return
     except subprocess.CalledProcessError as error:
         if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
@@ -55,7 +56,7 @@ def create_repository(
             else ('init',)
         )
         + (('--encryption', encryption_mode) if encryption_mode else ())
-        + (('--other-repo', key_repository) if key_repository else ())
+        + (('--other-repo', source_repository) if source_repository else ())
         + (('--copy-crypt-key',) if copy_crypt_key else ())
         + (('--append-only',) if append_only else ())
         + (('--storage-quota', storage_quota) if storage_quota else ())
@@ -67,6 +68,10 @@ def create_repository(
         + flags.make_repository_flags(repository, local_borg_version)
     )
 
+    if dry_run:
+        logging.info(f'{repository}: Skipping repository creation (dry run)')
+        return
+
     # Do not capture output here, so as to support interactive prompts.
     execute_command(
         rcreate_command,

borgmatic/borg/transfer.py

@@ -0,0 +1,45 @@
+import logging
+
+from borgmatic.borg import environment, flags
+from borgmatic.execute import execute_command
+
+logger = logging.getLogger(__name__)
+
+
+def transfer_archives(
+    dry_run,
+    repository,
+    storage_config,
+    local_borg_version,
+    transfer_arguments,
+    local_path='borg',
+    remote_path=None,
+):
+    '''
+    Given a dry-run flag, a local or remote repository path, a storage config dict, the local Borg
+    version, and the arguments to the transfer action, transfer archives to the given repository.
+    '''
+    full_command = (
+        (local_path, 'transfer')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('lock-wait', storage_config.get('lock_wait', None))
+        + flags.make_flags(
+            'glob-archives', transfer_arguments.glob_archives or transfer_arguments.archive
+        )
+        + flags.make_flags_from_arguments(
+            transfer_arguments,
+            excludes=('repository', 'source_repository', 'archive', 'glob_archives'),
+        )
+        + flags.make_repository_flags(repository, local_borg_version)
+        + flags.make_flags('other-repo', transfer_arguments.source_repository)
+        + flags.make_flags('dry-run', dry_run)
+    )
+
+    return execute_command(
+        full_command,
+        output_log_level=logging.WARNING,
+        borg_local_path=local_path,
+        extra_environment=environment.make_environment(storage_config),
+    )

borgmatic/commands/arguments.py

@@ -241,15 +241,15 @@ def make_parsers():
         required=True,
     )
     rcreate_group.add_argument(
-        '--key-repository',
+        '--source-repository',
         '--other-repo',
-        metavar='SOURCE_REPOSITORY',
+        metavar='KEY_REPOSITORY',
         help='Path to an existing Borg repository whose key material should be reused (Borg 2.x+ only)',
     )
     rcreate_group.add_argument(
         '--copy-crypt-key',
         action='store_true',
-        help='Copy the crypt key used for authenticated encryption from the key repository, defaults to a new random key (Borg 2.x+ only)',
+        help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key (Borg 2.x+ only)',
     )
     rcreate_group.add_argument(
         '--append-only', action='store_true', help='Create an append-only repository',
@@ -266,6 +266,53 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
+    transfer_parser = subparsers.add_parser(
+        'transfer',
+        aliases=SUBPARSER_ALIASES['transfer'],
+        help='Transfer archives from one repository to another, optionally upgrading the transferred data',
+        description='Transfer archives from one repository to another, optionally upgrading the transferred data',
+        add_help=False,
+    )
+    transfer_group = transfer_parser.add_argument_group('transfer arguments')
+    transfer_group.add_argument(
+        '--repository',
+        help='Path of existing destination repository to transfer archives to, defaults to the configured repository if there is only one',
+    )
+    transfer_group.add_argument(
+        '--source-repository',
+        help='Path of existing source repository to transfer archives from',
+        required=True,
+    )
+    transfer_group.add_argument(
+        '--archive',
+        help='Name of single archive to transfer (or "latest"), defaults to transferring all archives',
+    )
+    transfer_group.add_argument(
+        '--upgrader',
+        help='Upgrader type used to convert the transfered data, e.g. "From12To20" to upgrade data from Borg 1.2 to 2.0 format, defaults to no conversion',
+        required=True,
+    )
+    transfer_group.add_argument(
+        '-a',
+        '--glob-archives',
+        metavar='GLOB',
+        help='Only transfer archives with names matching this glob',
+    )
+    transfer_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    transfer_group.add_argument(
+        '--first',
+        metavar='N',
+        help='Only transfer first N archives after other filters are applied',
+    )
+    transfer_group.add_argument(
+        '--last', metavar='N', help='Only transfer last N archives after other filters are applied'
+    )
+    transfer_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
     prune_parser = subparsers.add_parser(
         'prune',
         aliases=SUBPARSER_ALIASES['prune'],
@@ -760,9 +807,6 @@ def parse_arguments(*unparsed_arguments):
             'The --excludes flag has been replaced with exclude_patterns in configuration.'
         )
 
-    if 'rcreate' in arguments and arguments['global'].dry_run:
-        raise ValueError('The rcreate/init action cannot be used with the --dry-run flag.')
-
     if (
         ('list' in arguments and 'rinfo' in arguments and arguments['list'].json)
         or ('list' in arguments and 'info' in arguments and arguments['list'].json)
@@ -770,6 +814,15 @@ def parse_arguments(*unparsed_arguments):
     ):
         raise ValueError('With the --json flag, multiple actions cannot be used together.')
 
+    if (
+        'transfer' in arguments
+        and arguments['transfer'].archive
+        and arguments['transfer'].glob_archives
+    ):
+        raise ValueError(
+            'With the transfer action, only one of --archive and --glob-archives flags can be used.'
+        )
+
     if 'info' in arguments and (
         (arguments['info'].archive and arguments['info'].prefix)
         or (arguments['info'].archive and arguments['info'].glob_archives)

borgmatic/commands/borgmatic.py

@@ -26,6 +26,7 @@ from borgmatic.borg import prune as borg_prune
 from borgmatic.borg import rcreate as borg_rcreate
 from borgmatic.borg import rinfo as borg_rinfo
 from borgmatic.borg import rlist as borg_rlist
+from borgmatic.borg import transfer as borg_transfer
 from borgmatic.borg import umount as borg_umount
 from borgmatic.borg import version as borg_version
 from borgmatic.commands.arguments import parse_arguments
@@ -254,11 +255,12 @@ def run_actions(
     if 'rcreate' in arguments:
         logger.info('{}: Creating repository'.format(repository))
         borg_rcreate.create_repository(
+            global_arguments.dry_run,
             repository,
             storage,
             local_borg_version,
             arguments['rcreate'].encryption_mode,
-            arguments['rcreate'].key_repository,
+            arguments['rcreate'].source_repository,
             arguments['rcreate'].copy_crypt_key,
             arguments['rcreate'].append_only,
             arguments['rcreate'].storage_quota,
@@ -266,6 +268,17 @@ def run_actions(
             local_path=local_path,
             remote_path=remote_path,
         )
+    if 'transfer' in arguments:
+        logger.info(f'{repository}: Transferring archives to repository')
+        borg_transfer.transfer_archives(
+            global_arguments.dry_run,
+            repository,
+            storage,
+            local_borg_version,
+            transfer_arguments=arguments['transfer'],
+            local_path=local_path,
+            remote_path=remote_path,
+        )
     if 'prune' in arguments:
         command.execute_hook(
             hooks.get('before_prune'),

docs/how-to/upgrade.md

@@ -1,11 +1,11 @@
 ---
-title: How to upgrade borgmatic
+title: How to upgrade borgmatic and Borg
 eleventyNavigation:
-  key: 📦 Upgrade borgmatic
+  key: 📦 Upgrade borgmatic/Borg
   parent: How-to guides
   order: 12
 ---
-## Upgrading
+## Upgrading borgmatic
 
 In general, all you should need to do to upgrade borgmatic is run the
 following:
@@ -115,3 +115,85 @@ sudo pip3 install --user borgmatic
 
 That's it! borgmatic will continue using your /etc/borgmatic configuration
 files.
+
+
+## Upgrading Borg
+
+To upgrade to a new version of Borg, you can generally install a new version
+the same way you installed the previous version, paying attention to any
+instructions included with each Borg release changelog linked from the
+[releases page](https://github.com/borgbackup/borg/releases). However, some
+more major Borg releases require additional steps that borgmatic can help
+with.
+
+
+### Borg 1.2 to 2.0
+
+<span class="minilink minilink-addedin">New in borgmatic version 1.7.0</span>
+Upgrading Borg from 1.2 to 2.0 requires manually upgrading your existing Borg
+1 repositories before use with Borg or borgmatic. Here's how you can
+accomplish that.
+
+Start by upgrading borgmatic as described above to at least version 1.7.0 and
+Borg to 2.0. Then, rename your repository in borgmatic's configuration file to
+a new repository path. The repository upgrade process does not occur
+in-place; you'll create a new repository with a copy of your old repository's
+data.
+
+Let's say your original borgmatic repository configuration file looks something
+like this:
+
+```yaml
+location:
+    repositories:
+        - original.borg
+```
+
+Change it to a new (not yet created) repository path:
+
+```yaml
+location:
+    repositories:
+        - upgraded.borg
+```
+
+Then, run the `rcreate` action (formerly `init`) to create that new Borg 2
+repository:
+
+```bash
+borgmatic rcreate --verbosity 1 --encryption repokey-aes-ocb \
+    --source-repository original.borg --repository upgraded.borg
+```
+
+(Note that `repokey-chacha20-poly1305` may be faster than `repokey-aes-ocb` on
+certain platforms like ARM64.)
+
+This creates an empty repository and doesn't actually transfer any data yet.
+The `--source-repository` flag is necessary to reuse key material from your
+Borg 1 repository so that the subsequent data transfer can work.
+
+To transfer data from your original Borg 1 repository to your newly created
+Borg 2 repository:
+
+```bash
+borgmatic transfer --verbosity 1 --upgrader From12To20 --source-repository \
+    original.borg --repository upgraded.borg --dry-run
+borgmatic transfer --verbosity 1 --upgrader From12To20 --source-repository \
+    original.borg --repository upgraded.borg
+borgmatic transfer --verbosity 1 --upgrader From12To20 --source-repository \
+    original.borg --repository upgraded.borg --dry-run
+```
+
+The first command with `--dry-run` tells you what Borg is going to do during
+the transfer, the second command actually performs the transfer/upgrade (this
+might take a while), and the final command with `--dry-run` again provides
+confirmation of success—or tells you if something hasn't been transferred yet.
+
+Note that by omitting the `--upgrader` flag, you can also do archive transfers
+between Borg 2 repositories without upgrading, even down to individual
+archives. For more on that functionality, see the [Borg transfer
+documentation](https://borgbackup.readthedocs.io/en/2.0.0b1/usage/transfer.html).
+
+That's it! Now you can use your new Borg 2 repository as normal with
+borgmatic. If you've got multiple repositories, repeat the above process for
+each.

tests/integration/commands/test_arguments.py

@@ -287,15 +287,6 @@ def test_parse_arguments_allows_init_and_create():
     module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'create')
 
 
-def test_parse_arguments_disallows_init_and_dry_run():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    with pytest.raises(ValueError):
-        module.parse_arguments(
-            '--config', 'myconfig', 'init', '--encryption', 'repokey', '--dry-run'
-        )
-
-
 def test_parse_arguments_disallows_repository_unless_action_consumes_it():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 

tests/unit/borg/test_rcreate.py

@@ -39,7 +39,26 @@ def test_create_repository_calls_borg_with_flags():
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
-        repository='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey'
+        dry_run=False,
+        repository='repo',
+        storage_config={},
+        local_borg_version='2.3.4',
+        encryption_mode='repokey',
+    )
+
+
+def test_create_repository_with_dry_run_skips_borg_call():
+    insert_rinfo_command_not_found_mock()
+    flexmock(module).should_receive('execute_command').never()
+    flexmock(module.feature).should_receive('available').and_return(True)
+    flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
+
+    module.create_repository(
+        dry_run=True,
+        repository='repo',
+        storage_config={},
+        local_borg_version='2.3.4',
+        encryption_mode='repokey',
     )
 
 
@@ -54,6 +73,7 @@ def test_create_repository_raises_for_borg_rcreate_error():
 
     with pytest.raises(subprocess.CalledProcessError):
         module.create_repository(
+            dry_run=False,
             repository='repo',
             storage_config={},
             local_borg_version='2.3.4',
@@ -67,7 +87,11 @@ def test_create_repository_skips_creation_when_repository_already_exists():
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
-        repository='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey'
+        dry_run=False,
+        repository='repo',
+        storage_config={},
+        local_borg_version='2.3.4',
+        encryption_mode='repokey',
     )
 
 
@@ -78,6 +102,7 @@ def test_create_repository_raises_for_unknown_rinfo_command_error():
 
     with pytest.raises(subprocess.CalledProcessError):
         module.create_repository(
+            dry_run=False,
             repository='repo',
             storage_config={},
             local_borg_version='2.3.4',
@@ -85,18 +110,19 @@ def test_create_repository_raises_for_unknown_rinfo_command_error():
         )
 
 
-def test_create_repository_with_key_repository_calls_borg_with_other_repo_flag():
+def test_create_repository_with_source_repository_calls_borg_with_other_repo_flag():
     insert_rinfo_command_not_found_mock()
     insert_rcreate_command_mock(RCREATE_COMMAND + ('--other-repo', 'other.borg', '--repo', 'repo'))
     flexmock(module.feature).should_receive('available').and_return(True)
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
+        dry_run=False,
         repository='repo',
         storage_config={},
         local_borg_version='2.3.4',
         encryption_mode='repokey',
-        key_repository='other.borg',
+        source_repository='other.borg',
     )
 
 
@@ -107,6 +133,7 @@ def test_create_repository_with_copy_crypt_key_calls_borg_with_copy_crypt_key_fl
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
+        dry_run=False,
         repository='repo',
         storage_config={},
         local_borg_version='2.3.4',
@@ -122,6 +149,7 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag():
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
+        dry_run=False,
         repository='repo',
         storage_config={},
         local_borg_version='2.3.4',
@@ -137,6 +165,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
+        dry_run=False,
         repository='repo',
         storage_config={},
         local_borg_version='2.3.4',
@@ -152,6 +181,7 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
+        dry_run=False,
         repository='repo',
         storage_config={},
         local_borg_version='2.3.4',
@@ -168,7 +198,11 @@ def test_create_repository_with_log_info_calls_borg_with_info_flag():
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
-        repository='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey'
+        dry_run=False,
+        repository='repo',
+        storage_config={},
+        local_borg_version='2.3.4',
+        encryption_mode='repokey',
     )
 
 
@@ -180,7 +214,11 @@ def test_create_repository_with_log_debug_calls_borg_with_debug_flag():
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
-        repository='repo', storage_config={}, local_borg_version='2.3.4', encryption_mode='repokey'
+        dry_run=False,
+        repository='repo',
+        storage_config={},
+        local_borg_version='2.3.4',
+        encryption_mode='repokey',
     )
 
 
@@ -191,6 +229,7 @@ def test_create_repository_with_local_path_calls_borg_via_local_path():
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
+        dry_run=False,
         repository='repo',
         storage_config={},
         local_borg_version='2.3.4',
@@ -206,6 +245,7 @@ def test_create_repository_with_remote_path_calls_borg_with_remote_path_flag():
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
+        dry_run=False,
         repository='repo',
         storage_config={},
         local_borg_version='2.3.4',
@@ -221,6 +261,7 @@ def test_create_repository_with_extra_borg_options_calls_borg_with_extra_options
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
 
     module.create_repository(
+        dry_run=False,
         repository='repo',
         storage_config={'extra_borg_options': {'rcreate': '--extra --options'}},
         local_borg_version='2.3.4',

tests/unit/commands/test_borgmatic.py

@@ -346,7 +346,7 @@ def test_run_actions_does_not_raise_for_rcreate_action():
         'global': flexmock(monitoring_verbosity=1, dry_run=False),
         'rcreate': flexmock(
             encryption_mode=flexmock(),
-            key_repository=flexmock(),
+            source_repository=flexmock(),
             copy_crypt_key=flexmock(),
             append_only=flexmock(),
             storage_quota=flexmock(),
@@ -371,6 +371,30 @@ def test_run_actions_does_not_raise_for_rcreate_action():
     )
 
 
+def test_run_actions_does_not_raise_for_transfer_action():
+    flexmock(module.borg_transfer).should_receive('transfer_archives')
+    arguments = {
+        'global': flexmock(monitoring_verbosity=1, dry_run=False),
+        'transfer': flexmock(),
+    }
+
+    list(
+        module.run_actions(
+            arguments=arguments,
+            config_filename='test.yaml',
+            location={'repositories': ['repo']},
+            storage={},
+            retention={},
+            consistency={},
+            hooks={},
+            local_path=None,
+            remote_path=None,
+            local_borg_version=None,
+            repository_path='repo',
+        )
+    )
+
+
 def test_run_actions_calls_hooks_for_prune_action():
     flexmock(module.borg_prune).should_receive('prune_archives')
     flexmock(module.command).should_receive('execute_hook').twice()