Add tests for compact action (#394).

borgmatic/borg/compact.py

@@ -9,7 +9,6 @@ def compact_segments(
     dry_run,
     repository,
     storage_config,
-    retention_config,
     local_path='borg',
     remote_path=None,
     progress=False,
@@ -17,8 +16,8 @@ def compact_segments(
     threshold=None,
 ):
     '''
-    Given dry-run flag, a local or remote repository path, a storage config dict, and a
+    Given dry-run flag, a local or remote repository path, and a storage config dict, compact Borg
-    retention config dict, compact Borg segments in a repository.
+    segments in a repository.
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)

borgmatic/commands/arguments.py

@@ -287,7 +287,9 @@ def parse_arguments(*unparsed_arguments):
         dest='threshold',
         help='Minimum saved space percentage threshold for compacting a segment, defaults to 10',
     )
-    compact_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+    compact_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
 
     create_parser = subparsers.add_parser(
         'create',

borgmatic/commands/borgmatic.py

@@ -337,7 +337,6 @@ def run_actions(
             global_arguments.dry_run,
             repository,
             storage,
-            retention,
             local_path=local_path,
             remote_path=remote_path,
             progress=arguments['compact'].progress,

borgmatic/config/schema.yaml

@@ -346,27 +346,27 @@ properties:
                     init:
                         type: string
                         description: |
-                            Extra command-line options to pass to "borg init".
+                          Extra command-line options to pass to "borg init".
                         example: "--make-parent-dirs"
                     prune:
                         type: string
                         description: |
-                            Extra command-line options to pass to "borg prune".
+                          Extra command-line options to pass to "borg prune".
                         example: "--save-space"
                     compact:
                         type: string
                         description: |
-                            Extra command-line options to pass to "borg compact".
+                          Extra command-line options to pass to "borg compact".
                         example: "--save-space"
                     create:
                         type: string
                         description: |
-                            Extra command-line options to pass to "borg create".
+                          Extra command-line options to pass to "borg create".
                         example: "--no-files-cache"
                     check:
                         type: string
                         description: |
-                            Extra command-line options to pass to "borg check".
+                          Extra command-line options to pass to "borg check".
                         example: "--save-space"
                 description: |
                     Additional options to pass directly to particular Borg

docs/Dockerfile

@@ -1,7 +1,7 @@
 FROM python:3.8-alpine3.12 as borgmatic
 
 COPY . /app
-RUN pip install --no-cache ruamel.yaml.clib==0.2.2 /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
+RUN pip install --no-cache ruamel.yaml.clib==0.2.6 /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
     && for action in init prune create check extract export-tar mount umount restore list info borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \

scripts/run-full-tests

@@ -10,11 +10,12 @@
 
 set -e
 
-apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools
+apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \
+    py3-ruamel.yaml.clib
 # If certain dependencies of black are available in this version of Alpine, install them.
 apk add --no-cache py3-typed-ast py3-regex || true
-python3 -m pip install --upgrade pip==21.3.1 setuptools==58.2.0
+python3 -m pip install --no-cache --upgrade pip==22.0.3 setuptools==60.8.1
-pip3 install tox==3.24.4
+pip3 install tox==3.24.5
 export COVERAGE_FILE=/tmp/.coverage
 tox --workdir /tmp/.tox --sitepackages
 tox --workdir /tmp/.tox --sitepackages -e end-to-end

test_requirements.txt

@@ -4,15 +4,15 @@ black==19.10b0; python_version >= '3.8'
 click==7.1.2; python_version >= '3.8'
 colorama==0.4.4
 coverage==5.3
-flake8==3.8.4
+flake8==4.0.1
 flexmock==0.10.4
 isort==5.9.1
 mccabe==0.6.1
 pluggy==0.13.1
 pathspec==0.8.1; python_version >= '3.8'
 py==1.10.0
-pycodestyle==2.6.0
+pycodestyle==2.8.0
-pyflakes==2.2.0
+pyflakes==2.4.0
 jsonschema==3.2.0
 pytest==6.2.5
 pytest-cov==3.0.0

tests/end-to-end/docker-compose.yaml

@@ -16,7 +16,7 @@ services:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: test
   tests:
-    image: alpine:3.13
+    image: alpine:3.15
     volumes:
       - "../..:/app:ro"
     tmpfs:

tests/unit/borg/test_compact.py

@@ -0,0 +1,112 @@
+import logging
+
+from flexmock import flexmock
+
+from borgmatic.borg import compact as module
+
+from ..test_verbosity import insert_logging_mock
+
+
+def insert_execute_command_mock(compact_command, output_log_level):
+    flexmock(module).should_receive('execute_command').with_args(
+        compact_command, output_log_level=output_log_level, borg_local_path=compact_command[0]
+    ).once()
+
+
+COMPACT_COMMAND = ('borg', 'compact')
+
+
+def test_compact_segments_calls_borg_with_parameters():
+    insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.WARNING)
+
+    module.compact_segments(dry_run=False, repository='repo', storage_config={})
+
+
+def test_compact_segments_with_log_info_calls_borg_with_info_parameter():
+    insert_execute_command_mock(COMPACT_COMMAND + ('--info', 'repo'), logging.WARNING)
+    insert_logging_mock(logging.INFO)
+
+    module.compact_segments(repository='repo', storage_config={}, dry_run=False)
+
+
+def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter():
+    insert_execute_command_mock(COMPACT_COMMAND + ('--debug', '--show-rc', 'repo'), logging.WARNING)
+    insert_logging_mock(logging.DEBUG)
+
+    module.compact_segments(repository='repo', storage_config={}, dry_run=False)
+
+
+def test_compact_segments_with_dry_run_calls_borg_with_dry_run_parameter():
+    insert_execute_command_mock(COMPACT_COMMAND + ('--dry-run', 'repo'), logging.WARNING)
+
+    module.compact_segments(repository='repo', storage_config={}, dry_run=True)
+
+
+def test_compact_segments_with_local_path_calls_borg_via_local_path():
+    insert_execute_command_mock(('borg1',) + COMPACT_COMMAND[1:] + ('repo',), logging.WARNING)
+
+    module.compact_segments(
+        dry_run=False, repository='repo', storage_config={}, local_path='borg1',
+    )
+
+
+def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameters():
+    insert_execute_command_mock(
+        COMPACT_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.WARNING
+    )
+
+    module.compact_segments(
+        dry_run=False, repository='repo', storage_config={}, remote_path='borg1',
+    )
+
+
+def test_compact_segments_with_progress_calls_borg_with_progress_parameter():
+    insert_execute_command_mock(COMPACT_COMMAND + ('--progress', 'repo'), logging.WARNING)
+
+    module.compact_segments(
+        dry_run=False, repository='repo', storage_config={}, progress=True,
+    )
+
+
+def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_parameter():
+    insert_execute_command_mock(COMPACT_COMMAND + ('--cleanup-commits', 'repo'), logging.WARNING)
+
+    module.compact_segments(
+        dry_run=False, repository='repo', storage_config={}, cleanup_commits=True,
+    )
+
+
+def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter():
+    insert_execute_command_mock(COMPACT_COMMAND + ('--threshold', '20', 'repo'), logging.WARNING)
+
+    module.compact_segments(
+        dry_run=False, repository='repo', storage_config={}, threshold=20,
+    )
+
+
+def test_compact_segments_with_umask_calls_borg_with_umask_parameters():
+    storage_config = {'umask': '077'}
+    insert_execute_command_mock(COMPACT_COMMAND + ('--umask', '077', 'repo'), logging.WARNING)
+
+    module.compact_segments(
+        dry_run=False, repository='repo', storage_config=storage_config,
+    )
+
+
+def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    storage_config = {'lock_wait': 5}
+    insert_execute_command_mock(COMPACT_COMMAND + ('--lock-wait', '5', 'repo'), logging.WARNING)
+
+    module.compact_segments(
+        dry_run=False, repository='repo', storage_config=storage_config,
+    )
+
+
+def test_compact_segments_with_extra_borg_options_calls_borg_with_extra_options():
+    insert_execute_command_mock(COMPACT_COMMAND + ('--extra', '--options', 'repo'), logging.WARNING)
+
+    module.compact_segments(
+        dry_run=False,
+        repository='repo',
+        storage_config={'extra_borg_options': {'compact': '--extra --options'}},
+    )

tests/unit/commands/test_borgmatic.py

@@ -33,6 +33,16 @@ def test_run_configuration_calls_hooks_for_prune_action():
     list(module.run_configuration('test.yaml', config, arguments))
 
 
+def test_run_configuration_calls_hooks_for_compact_action():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.command).should_receive('execute_hook').twice()
+    flexmock(module).should_receive('run_actions').and_return([])
+    config = {'location': {'repositories': ['foo']}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'compact': flexmock()}
+
+    list(module.run_configuration('test.yaml', config, arguments))
+
+
 def test_run_configuration_executes_and_calls_hooks_for_create_action():
     flexmock(module.borg_environment).should_receive('initialize')
     flexmock(module.command).should_receive('execute_hook').twice()