From 9582324c88eb5b1b41affd70bfee6dac74edb723 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 7 Feb 2022 23:29:44 -0800 Subject: [PATCH] Compact repository segments with new "borgmatic compact" action (#394). --- NEWS | 1 + borgmatic/borg/compact.py | 42 +++++++++++++++++++++++++++++++++ borgmatic/commands/arguments.py | 31 ++++++++++++++++++++++++ borgmatic/commands/borgmatic.py | 30 +++++++++++++++++++++++ borgmatic/config/schema.yaml | 23 ++++++++++++++++++ 5 files changed, 127 insertions(+) create mode 100644 borgmatic/borg/compact.py diff --git a/NEWS b/NEWS index 2f35702..fa2e0d6 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 1.5.23.dev0 + * #394: Compact repository segments with new "borgmatic compact" action. Borg 1.2+ only. * #480, #482: Fix traceback when a YAML validation error occurs. 1.5.22 diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py new file mode 100644 index 0000000..9c6f660 --- /dev/null +++ b/borgmatic/borg/compact.py @@ -0,0 +1,42 @@ +import logging + +from borgmatic.execute import execute_command + +logger = logging.getLogger(__name__) + + +def compact_segments( + dry_run, + repository, + storage_config, + retention_config, + local_path='borg', + remote_path=None, + progress=False, + cleanup_commits=False, + threshold=None, +): + ''' + Given dry-run flag, a local or remote repository path, a storage config dict, and a + retention config dict, compact Borg segments in a repository. + ''' + umask = storage_config.get('umask', None) + lock_wait = storage_config.get('lock_wait', None) + extra_borg_options = storage_config.get('extra_borg_options', {}).get('compact', '') + + full_command = ( + (local_path, 'compact') + + (('--remote-path', remote_path) if remote_path else ()) + + (('--umask', str(umask)) if umask else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + + (('--progress',) if progress else ()) + + (('--cleanup-commits',) if cleanup_commits else ()) + + (('--threshold', str(threshold)) if threshold else ()) + + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + + (('--dry-run',) if dry_run else ()) + + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + + (repository,) + ) + + execute_command(full_command, output_log_level=logging.WARNING, borg_local_path=local_path) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 7600c1a..90061f1 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -6,6 +6,7 @@ from borgmatic.config import collect SUBPARSER_ALIASES = { 'init': ['--init', '-I'], 'prune': ['--prune', '-p'], + 'compact': [], 'create': ['--create', '-C'], 'check': ['--check', '-k'], 'extract': ['--extract', '-x'], @@ -258,6 +259,36 @@ def parse_arguments(*unparsed_arguments): ) prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') + compact_parser = subparsers.add_parser( + 'compact', + aliases=SUBPARSER_ALIASES['compact'], + help='compact segments to free space (Borg 1.2+ only)', + description='compact segments to free space (Borg 1.2+ only)', + add_help=False, + ) + compact_group = compact_parser.add_argument_group('compact arguments') + compact_group.add_argument( + '--progress', + dest='progress', + default=False, + action='store_true', + help='Display progress as each segment is compacted', + ) + compact_group.add_argument( + '--cleanup-commits', + dest='cleanup_commits', + default=False, + action='store_true', + help='Cleanup commit-only 17-byte segment files left behind by Borg 1.1', + ) + compact_group.add_argument( + '--threshold', + type=int, + dest='threshold', + help='Minimum saved space percentage threshold for compacting a segment, defaults to 10', + ) + compact_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') + create_parser = subparsers.add_parser( 'create', aliases=SUBPARSER_ALIASES['create'], diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 255ee83..3b21b64 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -13,6 +13,7 @@ import pkg_resources from borgmatic.borg import borg as borg_borg from borgmatic.borg import check as borg_check +from borgmatic.borg import compact as borg_compact from borgmatic.borg import create as borg_create from borgmatic.borg import environment as borg_environment from borgmatic.borg import export_tar as borg_export_tar @@ -80,6 +81,14 @@ def run_configuration(config_filename, config, arguments): 'pre-prune', global_arguments.dry_run, ) + if 'compact' in arguments: + command.execute_hook( + hooks.get('before_compact'), + hooks.get('umask'), + config_filename, + 'pre-compact', + global_arguments.dry_run, + ) if 'create' in arguments: command.execute_hook( hooks.get('before_backup'), @@ -169,6 +178,14 @@ def run_configuration(config_filename, config, arguments): 'post-prune', global_arguments.dry_run, ) + if 'compact' in arguments: + command.execute_hook( + hooks.get('after_compact'), + hooks.get('umask'), + config_filename, + 'post-compact', + global_arguments.dry_run, + ) if 'create' in arguments: dispatch.call_hooks( 'remove_database_dumps', @@ -314,6 +331,19 @@ def run_actions( stats=arguments['prune'].stats, files=arguments['prune'].files, ) + if 'compact' in arguments: + logger.info('{}: Compacting segments{}'.format(repository, dry_run_label)) + borg_compact.compact_segments( + global_arguments.dry_run, + repository, + storage, + retention, + local_path=local_path, + remote_path=remote_path, + progress=arguments['compact'].progress, + cleanup_commits=arguments['compact'].cleanup_commits, + threshold=arguments['compact'].threshold, + ) if 'create' in arguments: logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) dispatch.call_hooks( diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 8b747a7..dba6b5f 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -353,6 +353,11 @@ properties: description: | Extra command-line options to pass to "borg prune". example: "--save-space" + compact: + type: string + description: | + Extra command-line options to pass to "borg compact". + example: "--save-space" create: type: string description: | @@ -522,6 +527,15 @@ properties: before pruning, run once per configuration file. example: - echo "Starting pruning." + before_compact: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute + before compaction, run once per configuration file. + example: + - echo "Starting compaction." before_check: type: array items: @@ -549,6 +563,15 @@ properties: after creating a backup, run once per configuration file. example: - echo "Finished a backup." + after_compact: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to execute + after compaction, run once per configuration file. + example: + - echo "Finished compaction." after_prune: type: array items: