Add "borgmatic export-tar" action to export an archive to a tar-formatted file or stream (#300).

This commit is contained in:
Dan Helfman 2020-07-22 21:53:55 -07:00
parent a23083f737
commit b3fd1be5f6
6 changed files with 365 additions and 2 deletions

3
NEWS
View file

@ -1,4 +1,5 @@
1.5.9.dev0
1.5.9
* #300: Add "borgmatic export-tar" action to export an archive to a tar-formatted file or stream.
* #339: Fix for intermittent timing-related test failure of logging function.
* Clarify database documentation about excluding named pipes and character/block devices to prevent
hangs.

View file

@ -0,0 +1,64 @@
import logging
import os
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
logger = logging.getLogger(__name__)
def export_tar_archive(
dry_run,
repository,
archive,
paths,
destination_path,
storage_config,
local_path='borg',
remote_path=None,
tar_filter=None,
files=False,
strip_components=None,
):
'''
Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
export from the archive, a destination path to export to, a storage configuration dict, optional
local and remote Borg paths, an optional filter program, whether to include per-file details,
and an optional number of path components to strip, export the archive into the given
destination path as a tar-formatted file.
If the destination path is "-", then stream the output to stdout instead of to a file.
'''
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
full_command = (
(local_path, 'export-tar')
+ (('--remote-path', remote_path) if remote_path else ())
+ (('--umask', str(umask)) if umask else ())
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+ (('--list',) if files else ())
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+ (('--dry-run',) if dry_run else ())
+ (('--tar-filter', tar_filter) if tar_filter else ())
+ (('--strip-components', str(strip_components)) if strip_components else ())
+ ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
+ (destination_path,)
+ (tuple(paths) if paths else ())
)
if files and logger.getEffectiveLevel() == logging.WARNING:
output_log_level = logging.WARNING
else:
output_log_level = logging.INFO
if dry_run:
logging.info('{}: Skipping export to tar file (dry run)'.format(repository))
return
execute_command(
full_command,
output_file=DO_NOT_CAPTURE if destination_path == '-' else None,
output_log_level=output_log_level,
borg_local_path=local_path,
)

View file

@ -9,6 +9,7 @@ SUBPARSER_ALIASES = {
'create': ['--create', '-C'],
'check': ['--check', '-k'],
'extract': ['--extract', '-x'],
'export-tar': ['--export-tar'],
'mount': ['--mount', '-m'],
'umount': ['--umount', '-u'],
'restore': ['--restore', '-r'],
@ -358,6 +359,52 @@ def parse_arguments(*unparsed_arguments):
'-h', '--help', action='help', help='Show this help message and exit'
)
export_tar_parser = subparsers.add_parser(
'export-tar',
aliases=SUBPARSER_ALIASES['export-tar'],
help='Export an archive to a tar-formatted file or stream',
description='Export an archive to a tar-formatted file or stream',
add_help=False,
)
export_tar_group = export_tar_parser.add_argument_group('export-tar arguments')
export_tar_group.add_argument(
'--repository',
help='Path of repository to export from, defaults to the configured repository if there is only one',
)
export_tar_group.add_argument(
'--archive', help='Name of archive to export (or "latest")', required=True
)
export_tar_group.add_argument(
'--path',
metavar='PATH',
nargs='+',
dest='paths',
help='Paths to export from archive, defaults to the entire archive',
)
export_tar_group.add_argument(
'--destination',
metavar='PATH',
dest='destination',
help='Path to destination export tar file, or "-" for stdout (but be careful about dirtying output with --verbosity or --files)',
required=True,
)
export_tar_group.add_argument(
'--tar-filter', help='Name of filter program to pipe data through'
)
export_tar_group.add_argument(
'--files', default=False, action='store_true', help='Show per-file details'
)
export_tar_group.add_argument(
'--strip-components',
type=int,
metavar='NUMBER',
dest='strip_components',
help='Number of leading path components to remove from each exported path. Skip paths with fewer elements',
)
export_tar_group.add_argument(
'-h', '--help', action='help', help='Show this help message and exit'
)
mount_parser = subparsers.add_parser(
'mount',
aliases=SUBPARSER_ALIASES['mount'],

View file

@ -12,6 +12,7 @@ import pkg_resources
from borgmatic.borg import check as borg_check
from borgmatic.borg import create as borg_create
from borgmatic.borg import environment as borg_environment
from borgmatic.borg import export_tar as borg_export_tar
from borgmatic.borg import extract as borg_extract
from borgmatic.borg import info as borg_info
from borgmatic.borg import init as borg_init
@ -347,6 +348,30 @@ def run_actions(
strip_components=arguments['extract'].strip_components,
progress=arguments['extract'].progress,
)
if 'export-tar' in arguments:
if arguments['export-tar'].repository is None or validate.repositories_match(
repository, arguments['export-tar'].repository
):
logger.info(
'{}: Exporting archive {} as tar file'.format(
repository, arguments['export-tar'].archive
)
)
borg_export_tar.export_tar_archive(
global_arguments.dry_run,
repository,
borg_list.resolve_archive_name(
repository, arguments['export-tar'].archive, storage, local_path, remote_path
),
arguments['export-tar'].paths,
arguments['export-tar'].destination,
storage,
local_path=local_path,
remote_path=remote_path,
tar_filter=arguments['export-tar'].tar_filter,
files=arguments['export-tar'].files,
strip_components=arguments['export-tar'].strip_components,
)
if 'mount' in arguments:
if arguments['mount'].repository is None or validate.repositories_match(
repository, arguments['mount'].repository

View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup
VERSION = '1.5.9.dev0'
VERSION = '1.5.9'
setup(

View file

@ -0,0 +1,226 @@
import logging
from flexmock import flexmock
from borgmatic.borg import export_tar as module
from ..test_verbosity import insert_logging_mock
def insert_execute_command_mock(
command, output_log_level=logging.INFO, borg_local_path='borg', capture=True
):
flexmock(module).should_receive('execute_command').with_args(
command,
output_file=None if capture else module.DO_NOT_CAPTURE,
output_log_level=output_log_level,
borg_local_path=borg_local_path,
).once()
def test_export_tar_archive_calls_borg_with_path_parameters():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(
('borg', 'export-tar', 'repo::archive', 'test.tar', 'path1', 'path2')
)
module.export_tar_archive(
dry_run=False,
repository='repo',
archive='archive',
paths=['path1', 'path2'],
destination_path='test.tar',
storage_config={},
)
def test_export_tar_archive_calls_borg_with_local_path_parameters():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(
('borg1', 'export-tar', 'repo::archive', 'test.tar'), borg_local_path='borg1'
)
module.export_tar_archive(
dry_run=False,
repository='repo',
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={},
local_path='borg1',
)
def test_export_tar_archive_calls_borg_with_remote_path_parameters():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(
('borg', 'export-tar', '--remote-path', 'borg1', 'repo::archive', 'test.tar')
)
module.export_tar_archive(
dry_run=False,
repository='repo',
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={},
remote_path='borg1',
)
def test_export_tar_archive_calls_borg_with_umask_parameters():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(
('borg', 'export-tar', '--umask', '0770', 'repo::archive', 'test.tar')
)
module.export_tar_archive(
dry_run=False,
repository='repo',
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={'umask': '0770'},
)
def test_export_tar_archive_calls_borg_with_lock_wait_parameters():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(
('borg', 'export-tar', '--lock-wait', '5', 'repo::archive', 'test.tar')
)
module.export_tar_archive(
dry_run=False,
repository='repo',
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={'lock_wait': '5'},
)
def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(('borg', 'export-tar', '--info', 'repo::archive', 'test.tar'))
insert_logging_mock(logging.INFO)
module.export_tar_archive(
dry_run=False,
repository='repo',
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={},
)
def test_export_tar_archive_with_log_debug_calls_borg_with_debug_parameters():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(
('borg', 'export-tar', '--debug', '--show-rc', 'repo::archive', 'test.tar')
)
insert_logging_mock(logging.DEBUG)
module.export_tar_archive(
dry_run=False,
repository='repo',
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={},
)
def test_export_tar_archive_calls_borg_with_dry_run_parameter():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
flexmock(module).should_receive('execute_command').never()
module.export_tar_archive(
dry_run=True,
repository='repo',
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={},
)
def test_export_tar_archive_calls_borg_with_tar_filter_parameters():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(
('borg', 'export-tar', '--tar-filter', 'bzip2', 'repo::archive', 'test.tar')
)
module.export_tar_archive(
dry_run=False,
repository='repo',
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={},
tar_filter='bzip2',
)
def test_export_tar_archive_calls_borg_with_list_parameter():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(
('borg', 'export-tar', '--list', 'repo::archive', 'test.tar'),
output_log_level=logging.WARNING,
)
module.export_tar_archive(
dry_run=False,
repository='repo',
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={},
files=True,
)
def test_export_tar_archive_calls_borg_with_strip_components_parameter():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(
('borg', 'export-tar', '--strip-components', '5', 'repo::archive', 'test.tar')
)
module.export_tar_archive(
dry_run=False,
repository='repo',
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={},
strip_components=5,
)
def test_export_tar_archive_skips_abspath_for_remote_repository_parameter():
flexmock(module.os.path).should_receive('abspath').never()
insert_execute_command_mock(('borg', 'export-tar', 'server:repo::archive', 'test.tar'))
module.export_tar_archive(
dry_run=False,
repository='server:repo',
archive='archive',
paths=None,
destination_path='test.tar',
storage_config={},
)
def test_export_tar_archive_calls_borg_with_stdout_destination_path():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(('borg', 'export-tar', 'repo::archive', '-'), capture=False)
module.export_tar_archive(
dry_run=False,
repository='repo',
archive='archive',
paths=None,
destination_path='-',
storage_config={},
)