Pass through several "borg list" flags (#193).

NEWS

@@ -1,3 +1,7 @@
+1.3.11.dev0
+ * #193: Pass through several "borg list" flags like --short, --format, --sort-by, --first, --last,
+   etc. via borgmatic list command-line flags.
+
 1.3.10
  * #198: Fix for Borg create error output not showing up at borgmatic verbosity level zero.
 

borgmatic/borg/flags.py

@@ -0,0 +1,31 @@
+import itertools
+
+
+def make_flags(name, value):
+    '''
+    Given a flag name and its value, return it formatted as Borg-compatible flags.
+    '''
+    if not value:
+        return ()
+
+    flag = '--{}'.format(name.replace('_', '-'))
+
+    if value is True:
+        return (flag,)
+
+    return (flag, str(value))
+
+
+def make_flags_from_arguments(arguments, excludes=()):
+    '''
+    Given borgmatic command-line arguments as an instance of argparse.Namespace, and optionally a
+    list of named arguments to exclude, generate and return the corresponding Borg command-line
+    flags as a tuple.
+    '''
+    return tuple(
+        itertools.chain.from_iterable(
+            make_flags(name, value=getattr(arguments, name))
+            for name in vars(arguments)
+            if name not in excludes and not name.startswith('_')
+        )
+    )

borgmatic/borg/list.py

@@ -1,27 +1,42 @@
 import logging
 
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
 from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
-def list_archives(
+def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
-    repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False
-):
     '''
-    Given a local or remote repository path and a storage config dict, display the output of listing
+    Given a local or remote repository path, a storage config dict, and the arguments to the list
-    Borg archives in the repository or return JSON output. Or, if an archive name is given, listing
+    action, display the output of listing Borg archives in the repository or return JSON output. Or,
-    the files in that archive.
+    if an archive name is given, listing the files in that archive.
     '''
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'list', '::'.join((repository, archive)) if archive else repository)
+        (
-        + (('--remote-path', remote_path) if remote_path else ())
+            local_path,
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
+            'list',
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
+            '::'.join((repository, list_arguments.archive))
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
+            if list_arguments.archive
-        + (('--json',) if json else ())
+            else repository,
+        )
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not list_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not list_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(list_arguments, excludes=('repository', 'archive'))
     )
 
-    return execute_command(full_command, output_log_level=None if json else logging.WARNING)
+    return execute_command(
+        full_command, output_log_level=None if list_arguments.json else logging.WARNING
+    )

borgmatic/commands/arguments.py

@@ -248,7 +248,7 @@ def parse_arguments(*unparsed_arguments):
         'list',
         aliases=SUBPARSER_ALIASES['list'],
         help='List archives',
-        description='List archives',
+        description='List archives or the contents of an archive',
         add_help=False,
     )
     list_group = list_parser.add_argument_group('list arguments')
@@ -258,7 +258,38 @@ def parse_arguments(*unparsed_arguments):
     )
     list_group.add_argument('--archive', help='Name of archive to operate on')
     list_group.add_argument(
-        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+        '--short', default=False, action='store_true', help='Output only archive or path names'
+    )
+    list_group.add_argument('--format', help='Format for file listing')
+    list_group.add_argument(
+        '--json', default=False, action='store_true', help='Output results as JSON'
+    )
+    list_group.add_argument(
+        '-P', '--prefix', help='Only list archive names starting with this prefix'
+    )
+    list_group.add_argument(
+        '-a', '--glob-archives', metavar='GLOB', help='Only list archive names matching this glob'
+    )
+    list_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    list_group.add_argument(
+        '--first', metavar='N', help='List first N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '--last', metavar='N', help='List first N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
+    )
+    list_group.add_argument(
+        '--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line'
+    )
+    list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
+    list_group.add_argument(
+        '--pattern-from',
+        metavar='FILENAME',
+        help='Include or exclude paths matching patterns from pattern file, one per line',
     )
     list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 

borgmatic/commands/borgmatic.py

@@ -171,10 +171,9 @@ def run_actions(
             json_output = borg_list.list_archives(
                 repository,
                 storage,
-                arguments['list'].archive,
+                list_arguments=arguments['list'],
                 local_path=local_path,
                 remote_path=remote_path,
-                json=arguments['list'].json,
             )
             if json_output:
                 yield json.loads(json_output)

scripts/find-unsupported-borg-options

@@ -48,6 +48,17 @@ for sub_command in prune create check list info; do
             | grep -v '^--stats$' \
             | grep -v '^--verbose$' \
             | grep -v '^--warning$' \
+            | grep -v '^--exclude' \
+            | grep -v '^--exclude-from' \
+            | grep -v '^--first' \
+            | grep -v '^--format' \
+            | grep -v '^--glob-archives' \
+            | grep -v '^--last' \
+            | grep -v '^--list-format' \
+            | grep -v '^--patterns-from' \
+            | grep -v '^--prefix' \
+            | grep -v '^--short' \
+            | grep -v '^--sort-by' \
             | grep -v '^-h$' \
             >> all_borg_flags
     done

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.3.10'
+VERSION = '1.3.11.dev0'
 
 
 setup(

tests/unit/borg/test_flags.py

@@ -0,0 +1,47 @@
+from flexmock import flexmock
+
+from borgmatic.borg import flags as module
+
+
+def test_make_flags_formats_string_value():
+    assert module.make_flags('foo', 'bar') == ('--foo', 'bar')
+
+
+def test_make_flags_formats_integer_value():
+    assert module.make_flags('foo', 3) == ('--foo', '3')
+
+
+def test_make_flags_formats_true_value():
+    assert module.make_flags('foo', True) == ('--foo',)
+
+
+def test_make_flags_omits_false_value():
+    assert module.make_flags('foo', False) == ()
+
+
+def test_make_flags_formats_name_with_underscore():
+    assert module.make_flags('posix_me_harder', 'okay') == ('--posix-me-harder', 'okay')
+
+
+def test_make_flags_from_arguments_flattens_multiple_arguments():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('make_flags').with_args('baz', 'quux').and_return(
+        ('baz', 'quux')
+    )
+    arguments = flexmock(foo='bar', baz='quux')
+
+    assert module.make_flags_from_arguments(arguments) == ('foo', 'bar', 'baz', 'quux')
+
+
+def test_make_flags_from_arguments_excludes_underscored_argument_names():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    arguments = flexmock(foo='bar', _baz='quux')
+
+    assert module.make_flags_from_arguments(arguments) == ('foo', 'bar')
+
+
+def test_make_flags_from_arguments_omits_excludes():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    arguments = flexmock(foo='bar', baz='quux')
+
+    assert module.make_flags_from_arguments(arguments, excludes=('baz', 'other')) == ('foo', 'bar')

tests/unit/borg/test_list.py

@@ -1,5 +1,6 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import list as module
@@ -14,7 +15,9 @@ def test_list_archives_calls_borg_with_parameters():
         LIST_COMMAND, output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_list_archives_with_log_info_calls_borg_with_info_parameter():
@@ -23,7 +26,9 @@ def test_list_archives_with_log_info_calls_borg_with_info_parameter():
     )
     insert_logging_mock(logging.INFO)
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
@@ -32,7 +37,9 @@ def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
     )
     insert_logging_mock(logging.INFO)
 
-    module.list_archives(repository='repo', storage_config={}, json=True)
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
+    )
 
 
 def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
@@ -41,7 +48,9 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
     )
     insert_logging_mock(logging.DEBUG)
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
@@ -50,7 +59,9 @@ def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
     )
     insert_logging_mock(logging.DEBUG)
 
-    module.list_archives(repository='repo', storage_config={}, json=True)
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
+    )
 
 
 def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
@@ -59,7 +70,11 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
         LIST_COMMAND + ('--lock-wait', '5'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config=storage_config)
+    module.list_archives(
+        repository='repo',
+        storage_config=storage_config,
+        list_arguments=flexmock(archive=None, json=False),
+    )
 
 
 def test_list_archives_with_archive_calls_borg_with_archive_parameter():
@@ -68,7 +83,11 @@ def test_list_archives_with_archive_calls_borg_with_archive_parameter():
         ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config=storage_config, archive='archive')
+    module.list_archives(
+        repository='repo',
+        storage_config=storage_config,
+        list_arguments=flexmock(archive='archive', json=False),
+    )
 
 
 def test_list_archives_with_local_path_calls_borg_via_local_path():
@@ -76,7 +95,12 @@ def test_list_archives_with_local_path_calls_borg_via_local_path():
         ('borg1',) + LIST_COMMAND[1:], output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config={}, local_path='borg1')
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False),
+        local_path='borg1',
+    )
 
 
 def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters():
@@ -84,7 +108,51 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters()
         LIST_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config={}, remote_path='borg1')
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False),
+        remote_path='borg1',
+    )
+
+
+def test_list_archives_with_short_calls_borg_with_short_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--short',), output_log_level=logging.WARNING
+    ).and_return('[]')
+
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, short=True),
+    )
+
+
+@pytest.mark.parametrize(
+    'argument_name',
+    (
+        'prefix',
+        'glob_archives',
+        'sort_by',
+        'first',
+        'last',
+        'exclude',
+        'exclude_from',
+        'pattern',
+        'pattern_from',
+    ),
+)
+def test_list_archives_passes_through_arguments_to_borg(argument_name):
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--' + argument_name.replace('_', '-'), 'value'),
+        output_log_level=logging.WARNING,
+    ).and_return('[]')
+
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, **{argument_name: 'value'}),
+    )
 
 
 def test_list_archives_with_json_calls_borg_with_json_parameter():
@@ -92,6 +160,8 @@ def test_list_archives_with_json_calls_borg_with_json_parameter():
         LIST_COMMAND + ('--json',), output_log_level=None
     ).and_return('[]')
 
-    json_output = module.list_archives(repository='repo', storage_config={}, json=True)
+    json_output = module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
+    )
 
     assert json_output == '[]'