List the files within an archive via --list --archive option (#140).

This commit is contained in:
Dan Helfman 2019-02-23 23:02:17 -08:00
parent 26071de2e7
commit 4272c6b077
8 changed files with 153 additions and 55 deletions

3
NEWS
View file

@ -1,3 +1,6 @@
1.2.17
* #140: List the files within an archive via --list --archive option.
1.2.16 1.2.16
* #119: Include a sample borgmatic configuration file in the documentation. * #119: Include a sample borgmatic configuration file in the documentation.
* #123: Support for Borg archive restoration via borgmatic --extract command-line flag. * #123: Support for Borg archive restoration via borgmatic --extract command-line flag.

View file

@ -5,15 +5,17 @@ import subprocess
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def list_archives(repository, storage_config, local_path='borg', remote_path=None, json=False): def list_archives(
repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False
):
''' '''
Given a local or remote repository path, and a storage config dict, Given a local or remote repository path and a storage config dict, list Borg archives in the
list Borg archives in the repository. repository. Or, if an archive name is given, list the files in that archive.
''' '''
lock_wait = storage_config.get('lock_wait', None) lock_wait = storage_config.get('lock_wait', None)
full_command = ( full_command = (
(local_path, 'list', repository) (local_path, 'list', '::'.join((repository, archive)) if archive else repository)
+ (('--remote-path', remote_path) if remote_path else ()) + (('--remote-path', remote_path) if remote_path else ())
+ (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())

View file

@ -102,27 +102,49 @@ def parse_arguments(*arguments):
help='Create a repository with a fixed storage quota', help='Create a repository with a fixed storage quota',
) )
prune_group = parser.add_argument_group('options for --prune')
stats_argument = prune_group.add_argument(
'--stats',
dest='stats',
default=False,
action='store_true',
help='Display statistics of archive',
)
create_group = parser.add_argument_group('options for --create') create_group = parser.add_argument_group('options for --create')
create_group.add_argument( progress_argument = create_group.add_argument(
'--progress', '--progress',
dest='progress', dest='progress',
default=False, default=False,
action='store_true', action='store_true',
help='Display progress for each file as it is backed up', help='Display progress for each file as it is processed',
)
create_group._group_actions.append(stats_argument)
json_argument = create_group.add_argument(
'--json', dest='json', default=False, action='store_true', help='Output results as JSON'
) )
extract_group = parser.add_argument_group('options for --extract') extract_group = parser.add_argument_group('options for --extract')
extract_group.add_argument( repository_argument = extract_group.add_argument(
'--repository', '--repository',
help='Path of repository to restore from, defaults to the configured repository if there is only one', help='Path of repository to use, defaults to the configured repository if there is only one',
) )
extract_group.add_argument('--archive', help='Name of archive to restore') archive_argument = extract_group.add_argument('--archive', help='Name of archive to operate on')
extract_group.add_argument( extract_group.add_argument(
'--restore-path', '--restore-path',
nargs='+', nargs='+',
dest='restore_paths', dest='restore_paths',
help='Paths to restore from archive, defaults to the entire archive', help='Paths to restore from archive, defaults to the entire archive',
) )
extract_group._group_actions.append(progress_argument)
list_group = parser.add_argument_group('options for --list')
list_group._group_actions.append(repository_argument)
list_group._group_actions.append(archive_argument)
list_group._group_actions.append(json_argument)
info_group = parser.add_argument_group('options for --info')
info_group._group_actions.append(json_argument)
common_group = parser.add_argument_group('common options') common_group = parser.add_argument_group('common options')
common_group.add_argument( common_group.add_argument(
@ -140,20 +162,6 @@ def parse_arguments(*arguments):
dest='excludes_filename', dest='excludes_filename',
help='Deprecated in favor of exclude_patterns within configuration', help='Deprecated in favor of exclude_patterns within configuration',
) )
common_group.add_argument(
'--stats',
dest='stats',
default=False,
action='store_true',
help='Display statistics of archive with --create or --prune option',
)
common_group.add_argument(
'--json',
dest='json',
default=False,
action='store_true',
help='Output results from the --create, --list, or --info options as json',
)
common_group.add_argument( common_group.add_argument(
'-n', '-n',
'--dry-run', '--dry-run',
@ -196,10 +204,15 @@ def parse_arguments(*arguments):
raise ValueError('The --encryption option is required with the --init option') raise ValueError('The --encryption option is required with the --init option')
if not args.extract: if not args.extract:
if args.repository: if not args.list:
raise ValueError('The --repository option can only be used with the --extract option') if args.repository:
if args.archive: raise ValueError(
raise ValueError('The --archive option can only be used with the --extract option') 'The --repository option can only be used with the --extract and --list options'
)
if args.archive:
raise ValueError(
'The --archive option can only be used with the --extract and --list options'
)
if args.restore_paths: if args.restore_paths:
raise ValueError('The --restore-path option can only be used with the --extract option') raise ValueError('The --restore-path option can only be used with the --extract option')
if args.extract and not args.archive: if args.extract and not args.archive:
@ -360,14 +373,20 @@ def _run_commands_on_repository(
progress=args.progress, progress=args.progress,
) )
if args.list: if args.list:
logger.info('{}: Listing archives'.format(repository)) if args.repository is None or repository == args.repository:
output = borg_list.list_archives( logger.info('{}: Listing archives'.format(repository))
repository, storage, local_path=local_path, remote_path=remote_path, json=args.json output = borg_list.list_archives(
) repository,
if args.json: storage,
json_results.append(json.loads(output)) args.archive,
else: local_path=local_path,
sys.stdout.write(output) remote_path=remote_path,
json=args.json,
)
if args.json:
json_results.append(json.loads(output))
else:
sys.stdout.write(output)
if args.info: if args.info:
logger.info('{}: Displaying summary info for archives'.format(repository)) logger.info('{}: Displaying summary info for archives'.format(repository))
output = borg_info.display_archives_info( output = borg_info.display_archives_info(
@ -388,6 +407,7 @@ def collect_configuration_run_summary_logs(config_filenames, args):
# Dict mapping from config filename to corresponding parsed config dict. # Dict mapping from config filename to corresponding parsed config dict.
configs = collections.OrderedDict() configs = collections.OrderedDict()
# Parse and load each configuration file.
for config_filename in config_filenames: for config_filename in config_filenames:
try: try:
logger.info('{}: Parsing configuration file'.format(config_filename)) logger.info('{}: Parsing configuration file'.format(config_filename))
@ -403,13 +423,15 @@ def collect_configuration_run_summary_logs(config_filenames, args):
) )
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error)) yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
if args.extract: # Run cross-file validation checks.
if args.extract or (args.list and args.archive):
try: try:
validate.guard_configuration_contains_repository(args.repository, configs) validate.guard_configuration_contains_repository(args.repository, configs)
except ValueError as error: except ValueError as error:
yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error)) yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
return return
# Execute the actions corresponding to each configuration file.
for config_filename, config in configs.items(): for config_filename, config in configs.items():
try: try:
run_configuration(config_filename, config, args) run_configuration(config_filename, config, args)

View file

@ -130,7 +130,7 @@ def guard_configuration_contains_repository(repository, configurations):
if count > 1: if count > 1:
raise ValueError( raise ValueError(
'Can\'t determine which repository to extract. Use --repository option to disambiguate'.format( 'Can\'t determine which repository to use. Use --repository option to disambiguate'.format(
repository repository
) )
) )

View file

@ -1,7 +1,7 @@
from setuptools import setup, find_packages from setuptools import setup, find_packages
VERSION = '1.2.16' VERSION = '1.2.17'
setup( setup(

View file

@ -142,14 +142,28 @@ def test_parse_arguments_disallows_init_and_dry_run():
) )
def test_parse_arguments_disallows_repository_without_extract(): def test_parse_arguments_disallows_repository_without_extract_or_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError): with pytest.raises(ValueError):
module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg') module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg')
def test_parse_arguments_disallows_archive_without_extract(): def test_parse_arguments_allows_repository_with_extract():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments(
'--config', 'myconfig', '--extract', '--repository', 'test.borg', '--archive', 'test'
)
def test_parse_arguments_allows_repository_with_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--config', 'myconfig', '--list', '--repository', 'test.borg')
def test_parse_arguments_disallows_archive_without_extract_or_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError): with pytest.raises(ValueError):
@ -169,6 +183,12 @@ def test_parse_arguments_allows_archive_with_extract():
module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test') module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test')
def test_parse_arguments_allows_archive_with_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--config', 'myconfig', '--list', '--archive', 'test')
def test_parse_arguments_requires_archive_with_extract(): def test_parse_arguments_requires_archive_with_extract():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
@ -177,51 +197,73 @@ def test_parse_arguments_requires_archive_with_extract():
def test_parse_arguments_allows_progress_and_create(): def test_parse_arguments_allows_progress_and_create():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--progress', '--create', '--list') module.parse_arguments('--progress', '--create', '--list')
def test_parse_arguments_allows_progress_and_extract(): def test_parse_arguments_allows_progress_and_extract():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--progress', '--extract', '--archive', 'test', '--list') module.parse_arguments('--progress', '--extract', '--archive', 'test', '--list')
def test_parse_arguments_disallows_progress_without_create(): def test_parse_arguments_disallows_progress_without_create():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError): with pytest.raises(ValueError):
module.parse_arguments('--progress', '--list') module.parse_arguments('--progress', '--list')
def test_parse_arguments_with_stats_and_create_flags_does_not_raise(): def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--stats', '--create', '--list') module.parse_arguments('--stats', '--create', '--list')
def test_parse_arguments_with_stats_and_prune_flags_does_not_raise(): def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--stats', '--prune', '--list') module.parse_arguments('--stats', '--prune', '--list')
def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error(): def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError): with pytest.raises(ValueError):
module.parse_arguments('--stats', '--list') module.parse_arguments('--stats', '--list')
def test_parse_arguments_with_just_stats_flag_does_not_raise(): def test_parse_arguments_with_just_stats_flag_does_not_raise():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--stats') module.parse_arguments('--stats')
def test_parse_arguments_allows_json_with_list_or_info(): def test_parse_arguments_allows_json_with_list_or_info():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--list', '--json') module.parse_arguments('--list', '--json')
module.parse_arguments('--info', '--json') module.parse_arguments('--info', '--json')
def test_parse_arguments_disallows_json_without_list_or_info(): def test_parse_arguments_disallows_json_without_list_or_info():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError): with pytest.raises(ValueError):
module.parse_arguments('--json') module.parse_arguments('--json')
def test_parse_arguments_disallows_json_with_both_list_and_info(): def test_parse_arguments_disallows_json_with_both_list_and_info():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(ValueError): with pytest.raises(ValueError):
module.parse_arguments('--list', '--info', '--json') module.parse_arguments('--list', '--info', '--json')
def test_borgmatic_version_matches_news_version(): def test_borgmatic_version_matches_news_version():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
borgmatic_version = subprocess.check_output(('borgmatic', '--version')).decode('ascii') borgmatic_version = subprocess.check_output(('borgmatic', '--version')).decode('ascii')
news_version = open('NEWS').readline() news_version = open('NEWS').readline()

View file

@ -34,10 +34,18 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
module.list_archives(repository='repo', storage_config={}) module.list_archives(repository='repo', storage_config={})
def test_list_archives_with_json_calls_borg_with_json_parameter(): def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
insert_subprocess_mock(LIST_COMMAND + ('--json',)) storage_config = {'lock_wait': 5}
insert_subprocess_mock(LIST_COMMAND + ('--lock-wait', '5'))
module.list_archives(repository='repo', storage_config={}, json=True) module.list_archives(repository='repo', storage_config=storage_config)
def test_list_archives_with_archive_calls_borg_with_archive_parameter():
storage_config = {}
insert_subprocess_mock(('borg', 'list', 'repo::archive'))
module.list_archives(repository='repo', storage_config=storage_config, archive='archive')
def test_list_archives_with_local_path_calls_borg_via_local_path(): def test_list_archives_with_local_path_calls_borg_via_local_path():
@ -52,8 +60,7 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters()
module.list_archives(repository='repo', storage_config={}, remote_path='borg1') module.list_archives(repository='repo', storage_config={}, remote_path='borg1')
def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): def test_list_archives_with_json_calls_borg_with_json_parameter():
storage_config = {'lock_wait': 5} insert_subprocess_mock(LIST_COMMAND + ('--json',))
insert_subprocess_mock(LIST_COMMAND + ('--lock-wait', '5'))
module.list_archives(repository='repo', storage_config=storage_config) module.list_archives(repository='repo', storage_config={}, json=True)

View file

@ -50,22 +50,22 @@ def test_run_commands_handles_multiple_json_outputs_in_array():
def test_collect_configuration_run_summary_logs_info_for_success(): def test_collect_configuration_run_summary_logs_info_for_success():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}}) flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module).should_receive('run_configuration') flexmock(module).should_receive('run_configuration')
args = flexmock(extract=False) args = flexmock(extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args)) logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.INFO) assert all(log for log in logs if log.levelno == module.logging.INFO)
def test_collect_configuration_run_summary_logs_info_for_success_with_extract(): def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}}) flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module.validate).should_receive('guard_configuration_contains_repository') flexmock(module.validate).should_receive('guard_configuration_contains_repository')
flexmock(module).should_receive('run_configuration') flexmock(module).should_receive('run_configuration')
args = flexmock(extract=True, repository='repo') args = flexmock(extract=True, list=False, repository='repo')
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args)) logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.INFO) assert all(log for log in logs if log.levelno == module.logging.INFO)
def test_collect_configuration_run_summary_logs_critical_for_extract_with_repository_error(): def test_collect_configuration_run_summary_logs_critical_for_extract_with_repository_error():
@ -73,16 +73,38 @@ def test_collect_configuration_run_summary_logs_critical_for_extract_with_reposi
flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise( flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
ValueError ValueError
) )
args = flexmock(extract=True, repository='repo') args = flexmock(extract=True, list=False, repository='repo')
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args)) logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.CRITICAL) assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_and_repository_error():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
ValueError
)
args = flexmock(extract=False, list=True, repository='repo', archive='test')
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
def test_collect_configuration_run_summary_logs_info_for_success_with_list():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module).should_receive('run_configuration')
args = flexmock(extract=False, list=True, repository='repo', archive=None)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
assert all(log for log in logs if log.levelno == module.logging.INFO)
def test_collect_configuration_run_summary_logs_critical_for_parse_error(): def test_collect_configuration_run_summary_logs_critical_for_parse_error():
flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError) flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError)
args = flexmock(extract=False) args = flexmock(extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args)) logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
@ -93,7 +115,7 @@ def test_collect_configuration_run_summary_logs_critical_for_run_error():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}}) flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module.validate).should_receive('guard_configuration_contains_repository') flexmock(module.validate).should_receive('guard_configuration_contains_repository')
flexmock(module).should_receive('run_configuration').and_raise(ValueError) flexmock(module).should_receive('run_configuration').and_raise(ValueError)
args = flexmock(extract=False) args = flexmock(extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args)) logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
@ -103,7 +125,7 @@ def test_collect_configuration_run_summary_logs_critical_for_run_error():
def test_collect_configuration_run_summary_logs_critical_for_missing_configs(): def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}}) flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
flexmock(module).should_receive('run_configuration') flexmock(module).should_receive('run_configuration')
args = flexmock(config_paths=(), extract=False) args = flexmock(config_paths=(), extract=False, list=False)
logs = tuple(module.collect_configuration_run_summary_logs(config_filenames=(), args=args)) logs = tuple(module.collect_configuration_run_summary_logs(config_filenames=(), args=args))