From b3fd1be5f65a81381fb6ec8d4087e944efa67756 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 22 Jul 2020 21:53:55 -0700 Subject: [PATCH] Add "borgmatic export-tar" action to export an archive to a tar-formatted file or stream (#300). --- NEWS | 3 +- borgmatic/borg/export_tar.py | 64 ++++++++ borgmatic/commands/arguments.py | 47 ++++++ borgmatic/commands/borgmatic.py | 25 ++++ setup.py | 2 +- tests/unit/borg/test_export_tar.py | 226 +++++++++++++++++++++++++++++ 6 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 borgmatic/borg/export_tar.py create mode 100644 tests/unit/borg/test_export_tar.py diff --git a/NEWS b/NEWS index da8088e..3d473c7 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ -1.5.9.dev0 +1.5.9 + * #300: Add "borgmatic export-tar" action to export an archive to a tar-formatted file or stream. * #339: Fix for intermittent timing-related test failure of logging function. * Clarify database documentation about excluding named pipes and character/block devices to prevent hangs. diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py new file mode 100644 index 0000000..f0470a9 --- /dev/null +++ b/borgmatic/borg/export_tar.py @@ -0,0 +1,64 @@ +import logging +import os + +from borgmatic.execute import DO_NOT_CAPTURE, execute_command + +logger = logging.getLogger(__name__) + + +def export_tar_archive( + dry_run, + repository, + archive, + paths, + destination_path, + storage_config, + local_path='borg', + remote_path=None, + tar_filter=None, + files=False, + strip_components=None, +): + ''' + Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to + export from the archive, a destination path to export to, a storage configuration dict, optional + local and remote Borg paths, an optional filter program, whether to include per-file details, + and an optional number of path components to strip, export the archive into the given + destination path as a tar-formatted file. + + If the destination path is "-", then stream the output to stdout instead of to a file. + ''' + umask = storage_config.get('umask', None) + lock_wait = storage_config.get('lock_wait', None) + + full_command = ( + (local_path, 'export-tar') + + (('--remote-path', remote_path) if remote_path else ()) + + (('--umask', str(umask)) if umask else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + + (('--list',) if files else ()) + + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + + (('--dry-run',) if dry_run else ()) + + (('--tar-filter', tar_filter) if tar_filter else ()) + + (('--strip-components', str(strip_components)) if strip_components else ()) + + ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),) + + (destination_path,) + + (tuple(paths) if paths else ()) + ) + + if files and logger.getEffectiveLevel() == logging.WARNING: + output_log_level = logging.WARNING + else: + output_log_level = logging.INFO + + if dry_run: + logging.info('{}: Skipping export to tar file (dry run)'.format(repository)) + return + + execute_command( + full_command, + output_file=DO_NOT_CAPTURE if destination_path == '-' else None, + output_log_level=output_log_level, + borg_local_path=local_path, + ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 31a5ba6..c5dde7c 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -9,6 +9,7 @@ SUBPARSER_ALIASES = { 'create': ['--create', '-C'], 'check': ['--check', '-k'], 'extract': ['--extract', '-x'], + 'export-tar': ['--export-tar'], 'mount': ['--mount', '-m'], 'umount': ['--umount', '-u'], 'restore': ['--restore', '-r'], @@ -358,6 +359,52 @@ def parse_arguments(*unparsed_arguments): '-h', '--help', action='help', help='Show this help message and exit' ) + export_tar_parser = subparsers.add_parser( + 'export-tar', + aliases=SUBPARSER_ALIASES['export-tar'], + help='Export an archive to a tar-formatted file or stream', + description='Export an archive to a tar-formatted file or stream', + add_help=False, + ) + export_tar_group = export_tar_parser.add_argument_group('export-tar arguments') + export_tar_group.add_argument( + '--repository', + help='Path of repository to export from, defaults to the configured repository if there is only one', + ) + export_tar_group.add_argument( + '--archive', help='Name of archive to export (or "latest")', required=True + ) + export_tar_group.add_argument( + '--path', + metavar='PATH', + nargs='+', + dest='paths', + help='Paths to export from archive, defaults to the entire archive', + ) + export_tar_group.add_argument( + '--destination', + metavar='PATH', + dest='destination', + help='Path to destination export tar file, or "-" for stdout (but be careful about dirtying output with --verbosity or --files)', + required=True, + ) + export_tar_group.add_argument( + '--tar-filter', help='Name of filter program to pipe data through' + ) + export_tar_group.add_argument( + '--files', default=False, action='store_true', help='Show per-file details' + ) + export_tar_group.add_argument( + '--strip-components', + type=int, + metavar='NUMBER', + dest='strip_components', + help='Number of leading path components to remove from each exported path. Skip paths with fewer elements', + ) + export_tar_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' + ) + mount_parser = subparsers.add_parser( 'mount', aliases=SUBPARSER_ALIASES['mount'], diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 3420d30..a882531 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -12,6 +12,7 @@ import pkg_resources from borgmatic.borg import check as borg_check from borgmatic.borg import create as borg_create from borgmatic.borg import environment as borg_environment +from borgmatic.borg import export_tar as borg_export_tar from borgmatic.borg import extract as borg_extract from borgmatic.borg import info as borg_info from borgmatic.borg import init as borg_init @@ -347,6 +348,30 @@ def run_actions( strip_components=arguments['extract'].strip_components, progress=arguments['extract'].progress, ) + if 'export-tar' in arguments: + if arguments['export-tar'].repository is None or validate.repositories_match( + repository, arguments['export-tar'].repository + ): + logger.info( + '{}: Exporting archive {} as tar file'.format( + repository, arguments['export-tar'].archive + ) + ) + borg_export_tar.export_tar_archive( + global_arguments.dry_run, + repository, + borg_list.resolve_archive_name( + repository, arguments['export-tar'].archive, storage, local_path, remote_path + ), + arguments['export-tar'].paths, + arguments['export-tar'].destination, + storage, + local_path=local_path, + remote_path=remote_path, + tar_filter=arguments['export-tar'].tar_filter, + files=arguments['export-tar'].files, + strip_components=arguments['export-tar'].strip_components, + ) if 'mount' in arguments: if arguments['mount'].repository is None or validate.repositories_match( repository, arguments['mount'].repository diff --git a/setup.py b/setup.py index 3262f03..46073c5 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.5.9.dev0' +VERSION = '1.5.9' setup( diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py new file mode 100644 index 0000000..8a60b21 --- /dev/null +++ b/tests/unit/borg/test_export_tar.py @@ -0,0 +1,226 @@ +import logging + +from flexmock import flexmock + +from borgmatic.borg import export_tar as module + +from ..test_verbosity import insert_logging_mock + + +def insert_execute_command_mock( + command, output_log_level=logging.INFO, borg_local_path='borg', capture=True +): + flexmock(module).should_receive('execute_command').with_args( + command, + output_file=None if capture else module.DO_NOT_CAPTURE, + output_log_level=output_log_level, + borg_local_path=borg_local_path, + ).once() + + +def test_export_tar_archive_calls_borg_with_path_parameters(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock( + ('borg', 'export-tar', 'repo::archive', 'test.tar', 'path1', 'path2') + ) + + module.export_tar_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=['path1', 'path2'], + destination_path='test.tar', + storage_config={}, + ) + + +def test_export_tar_archive_calls_borg_with_local_path_parameters(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock( + ('borg1', 'export-tar', 'repo::archive', 'test.tar'), borg_local_path='borg1' + ) + + module.export_tar_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + local_path='borg1', + ) + + +def test_export_tar_archive_calls_borg_with_remote_path_parameters(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock( + ('borg', 'export-tar', '--remote-path', 'borg1', 'repo::archive', 'test.tar') + ) + + module.export_tar_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + remote_path='borg1', + ) + + +def test_export_tar_archive_calls_borg_with_umask_parameters(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock( + ('borg', 'export-tar', '--umask', '0770', 'repo::archive', 'test.tar') + ) + + module.export_tar_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={'umask': '0770'}, + ) + + +def test_export_tar_archive_calls_borg_with_lock_wait_parameters(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock( + ('borg', 'export-tar', '--lock-wait', '5', 'repo::archive', 'test.tar') + ) + + module.export_tar_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={'lock_wait': '5'}, + ) + + +def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock(('borg', 'export-tar', '--info', 'repo::archive', 'test.tar')) + insert_logging_mock(logging.INFO) + + module.export_tar_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + ) + + +def test_export_tar_archive_with_log_debug_calls_borg_with_debug_parameters(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock( + ('borg', 'export-tar', '--debug', '--show-rc', 'repo::archive', 'test.tar') + ) + insert_logging_mock(logging.DEBUG) + + module.export_tar_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + ) + + +def test_export_tar_archive_calls_borg_with_dry_run_parameter(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + flexmock(module).should_receive('execute_command').never() + + module.export_tar_archive( + dry_run=True, + repository='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + ) + + +def test_export_tar_archive_calls_borg_with_tar_filter_parameters(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock( + ('borg', 'export-tar', '--tar-filter', 'bzip2', 'repo::archive', 'test.tar') + ) + + module.export_tar_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + tar_filter='bzip2', + ) + + +def test_export_tar_archive_calls_borg_with_list_parameter(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock( + ('borg', 'export-tar', '--list', 'repo::archive', 'test.tar'), + output_log_level=logging.WARNING, + ) + + module.export_tar_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + files=True, + ) + + +def test_export_tar_archive_calls_borg_with_strip_components_parameter(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock( + ('borg', 'export-tar', '--strip-components', '5', 'repo::archive', 'test.tar') + ) + + module.export_tar_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + strip_components=5, + ) + + +def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): + flexmock(module.os.path).should_receive('abspath').never() + insert_execute_command_mock(('borg', 'export-tar', 'server:repo::archive', 'test.tar')) + + module.export_tar_archive( + dry_run=False, + repository='server:repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + ) + + +def test_export_tar_archive_calls_borg_with_stdout_destination_path(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock(('borg', 'export-tar', 'repo::archive', '-'), capture=False) + + module.export_tar_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + destination_path='-', + storage_config={}, + )