Compact repository segments with new "borgmatic compact" action (#394).

This commit is contained in:
Dan Helfman 2022-02-07 23:29:44 -08:00
parent bb0716421d
commit 9582324c88
5 changed files with 127 additions and 0 deletions

1
NEWS
View file

@ -1,4 +1,5 @@
1.5.23.dev0 1.5.23.dev0
* #394: Compact repository segments with new "borgmatic compact" action. Borg 1.2+ only.
* #480, #482: Fix traceback when a YAML validation error occurs. * #480, #482: Fix traceback when a YAML validation error occurs.
1.5.22 1.5.22

42
borgmatic/borg/compact.py Normal file
View file

@ -0,0 +1,42 @@
import logging
from borgmatic.execute import execute_command
logger = logging.getLogger(__name__)
def compact_segments(
dry_run,
repository,
storage_config,
retention_config,
local_path='borg',
remote_path=None,
progress=False,
cleanup_commits=False,
threshold=None,
):
'''
Given dry-run flag, a local or remote repository path, a storage config dict, and a
retention config dict, compact Borg segments in a repository.
'''
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
extra_borg_options = storage_config.get('extra_borg_options', {}).get('compact', '')
full_command = (
(local_path, 'compact')
+ (('--remote-path', remote_path) if remote_path else ())
+ (('--umask', str(umask)) if umask else ())
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
+ (('--progress',) if progress else ())
+ (('--cleanup-commits',) if cleanup_commits else ())
+ (('--threshold', str(threshold)) if threshold else ())
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+ (('--dry-run',) if dry_run else ())
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+ (repository,)
)
execute_command(full_command, output_log_level=logging.WARNING, borg_local_path=local_path)

View file

@ -6,6 +6,7 @@ from borgmatic.config import collect
SUBPARSER_ALIASES = { SUBPARSER_ALIASES = {
'init': ['--init', '-I'], 'init': ['--init', '-I'],
'prune': ['--prune', '-p'], 'prune': ['--prune', '-p'],
'compact': [],
'create': ['--create', '-C'], 'create': ['--create', '-C'],
'check': ['--check', '-k'], 'check': ['--check', '-k'],
'extract': ['--extract', '-x'], 'extract': ['--extract', '-x'],
@ -258,6 +259,36 @@ def parse_arguments(*unparsed_arguments):
) )
prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
compact_parser = subparsers.add_parser(
'compact',
aliases=SUBPARSER_ALIASES['compact'],
help='compact segments to free space (Borg 1.2+ only)',
description='compact segments to free space (Borg 1.2+ only)',
add_help=False,
)
compact_group = compact_parser.add_argument_group('compact arguments')
compact_group.add_argument(
'--progress',
dest='progress',
default=False,
action='store_true',
help='Display progress as each segment is compacted',
)
compact_group.add_argument(
'--cleanup-commits',
dest='cleanup_commits',
default=False,
action='store_true',
help='Cleanup commit-only 17-byte segment files left behind by Borg 1.1',
)
compact_group.add_argument(
'--threshold',
type=int,
dest='threshold',
help='Minimum saved space percentage threshold for compacting a segment, defaults to 10',
)
compact_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
create_parser = subparsers.add_parser( create_parser = subparsers.add_parser(
'create', 'create',
aliases=SUBPARSER_ALIASES['create'], aliases=SUBPARSER_ALIASES['create'],

View file

@ -13,6 +13,7 @@ import pkg_resources
from borgmatic.borg import borg as borg_borg from borgmatic.borg import borg as borg_borg
from borgmatic.borg import check as borg_check from borgmatic.borg import check as borg_check
from borgmatic.borg import compact as borg_compact
from borgmatic.borg import create as borg_create from borgmatic.borg import create as borg_create
from borgmatic.borg import environment as borg_environment from borgmatic.borg import environment as borg_environment
from borgmatic.borg import export_tar as borg_export_tar from borgmatic.borg import export_tar as borg_export_tar
@ -80,6 +81,14 @@ def run_configuration(config_filename, config, arguments):
'pre-prune', 'pre-prune',
global_arguments.dry_run, global_arguments.dry_run,
) )
if 'compact' in arguments:
command.execute_hook(
hooks.get('before_compact'),
hooks.get('umask'),
config_filename,
'pre-compact',
global_arguments.dry_run,
)
if 'create' in arguments: if 'create' in arguments:
command.execute_hook( command.execute_hook(
hooks.get('before_backup'), hooks.get('before_backup'),
@ -169,6 +178,14 @@ def run_configuration(config_filename, config, arguments):
'post-prune', 'post-prune',
global_arguments.dry_run, global_arguments.dry_run,
) )
if 'compact' in arguments:
command.execute_hook(
hooks.get('after_compact'),
hooks.get('umask'),
config_filename,
'post-compact',
global_arguments.dry_run,
)
if 'create' in arguments: if 'create' in arguments:
dispatch.call_hooks( dispatch.call_hooks(
'remove_database_dumps', 'remove_database_dumps',
@ -314,6 +331,19 @@ def run_actions(
stats=arguments['prune'].stats, stats=arguments['prune'].stats,
files=arguments['prune'].files, files=arguments['prune'].files,
) )
if 'compact' in arguments:
logger.info('{}: Compacting segments{}'.format(repository, dry_run_label))
borg_compact.compact_segments(
global_arguments.dry_run,
repository,
storage,
retention,
local_path=local_path,
remote_path=remote_path,
progress=arguments['compact'].progress,
cleanup_commits=arguments['compact'].cleanup_commits,
threshold=arguments['compact'].threshold,
)
if 'create' in arguments: if 'create' in arguments:
logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
dispatch.call_hooks( dispatch.call_hooks(

View file

@ -353,6 +353,11 @@ properties:
description: | description: |
Extra command-line options to pass to "borg prune". Extra command-line options to pass to "borg prune".
example: "--save-space" example: "--save-space"
compact:
type: string
description: |
Extra command-line options to pass to "borg compact".
example: "--save-space"
create: create:
type: string type: string
description: | description: |
@ -522,6 +527,15 @@ properties:
before pruning, run once per configuration file. before pruning, run once per configuration file.
example: example:
- echo "Starting pruning." - echo "Starting pruning."
before_compact:
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
before compaction, run once per configuration file.
example:
- echo "Starting compaction."
before_check: before_check:
type: array type: array
items: items:
@ -549,6 +563,15 @@ properties:
after creating a backup, run once per configuration file. after creating a backup, run once per configuration file.
example: example:
- echo "Finished a backup." - echo "Finished a backup."
after_compact:
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
after compaction, run once per configuration file.
example:
- echo "Finished compaction."
after_prune: after_prune:
type: array type: array
items: items: