Pass extra options directly to particular Borg commands, handy for Borg options that borgmatic does not yet support natively (#235).
This commit is contained in:
parent
00f62ca023
commit
0c6c61a272
12 changed files with 130 additions and 14 deletions
5
NEWS
5
NEWS
|
@ -1,3 +1,8 @@
|
||||||
|
1.4.17
|
||||||
|
* #235: Pass extra options directly to particular Borg commands, handy for Borg options that
|
||||||
|
borgmatic does not yet support natively. Use "extra_borg_options" in the storage configuration
|
||||||
|
section.
|
||||||
|
|
||||||
1.4.16
|
1.4.16
|
||||||
* #256: Fix for "before_backup" hook not triggering an error when the command contains "borg" and
|
* #256: Fix for "before_backup" hook not triggering an error when the command contains "borg" and
|
||||||
has an exit code of 1.
|
has an exit code of 1.
|
||||||
|
|
|
@ -103,6 +103,7 @@ def check_archives(
|
||||||
checks = _parse_checks(consistency_config, only_checks)
|
checks = _parse_checks(consistency_config, only_checks)
|
||||||
check_last = consistency_config.get('check_last', None)
|
check_last = consistency_config.get('check_last', None)
|
||||||
lock_wait = None
|
lock_wait = None
|
||||||
|
extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
|
||||||
|
|
||||||
if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))):
|
if set(checks).intersection(set(DEFAULT_CHECKS + ('data',))):
|
||||||
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
||||||
|
@ -123,6 +124,7 @@ def check_archives(
|
||||||
+ remote_path_flags
|
+ remote_path_flags
|
||||||
+ lock_wait_flags
|
+ lock_wait_flags
|
||||||
+ verbosity_flags
|
+ verbosity_flags
|
||||||
|
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
|
||||||
+ (repository,)
|
+ (repository,)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -150,6 +150,7 @@ def create_archive(
|
||||||
files_cache = location_config.get('files_cache')
|
files_cache = location_config.get('files_cache')
|
||||||
default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
|
default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
|
||||||
archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
|
archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
|
||||||
|
extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
|
||||||
|
|
||||||
full_command = (
|
full_command = (
|
||||||
(local_path, 'create')
|
(local_path, 'create')
|
||||||
|
@ -185,6 +186,7 @@ def create_archive(
|
||||||
+ (('--dry-run',) if dry_run else ())
|
+ (('--dry-run',) if dry_run else ())
|
||||||
+ (('--progress',) if progress else ())
|
+ (('--progress',) if progress else ())
|
||||||
+ (('--json',) if json else ())
|
+ (('--json',) if json else ())
|
||||||
|
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
|
||||||
+ (
|
+ (
|
||||||
'{repository}::{archive_name_format}'.format(
|
'{repository}::{archive_name_format}'.format(
|
||||||
repository=repository, archive_name_format=archive_name_format
|
repository=repository, archive_name_format=archive_name_format
|
||||||
|
|
|
@ -11,6 +11,7 @@ INFO_REPOSITORY_NOT_FOUND_EXIT_CODE = 2
|
||||||
|
|
||||||
def initialize_repository(
|
def initialize_repository(
|
||||||
repository,
|
repository,
|
||||||
|
storage_config,
|
||||||
encryption_mode,
|
encryption_mode,
|
||||||
append_only=None,
|
append_only=None,
|
||||||
storage_quota=None,
|
storage_quota=None,
|
||||||
|
@ -18,9 +19,9 @@ def initialize_repository(
|
||||||
remote_path=None,
|
remote_path=None,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given a local or remote repository path, a Borg encryption mode, whether the repository should
|
Given a local or remote repository path, a storage configuration dict, a Borg encryption mode,
|
||||||
be append-only, and the storage quota to use, initialize the repository. If the repository
|
whether the repository should be append-only, and the storage quota to use, initialize the
|
||||||
already exists, then log and skip initialization.
|
repository. If the repository already exists, then log and skip initialization.
|
||||||
'''
|
'''
|
||||||
info_command = (local_path, 'info', repository)
|
info_command = (local_path, 'info', repository)
|
||||||
logger.debug(' '.join(info_command))
|
logger.debug(' '.join(info_command))
|
||||||
|
@ -33,6 +34,8 @@ def initialize_repository(
|
||||||
if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
|
if error.returncode != INFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
extra_borg_options = storage_config.get('extra_borg_options', {}).get('init', '')
|
||||||
|
|
||||||
init_command = (
|
init_command = (
|
||||||
(local_path, 'init')
|
(local_path, 'init')
|
||||||
+ (('--encryption', encryption_mode) if encryption_mode else ())
|
+ (('--encryption', encryption_mode) if encryption_mode else ())
|
||||||
|
@ -41,6 +44,7 @@ def initialize_repository(
|
||||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||||
+ (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
|
+ (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
|
||||||
+ (('--remote-path', remote_path) if remote_path else ())
|
+ (('--remote-path', remote_path) if remote_path else ())
|
||||||
|
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
|
||||||
+ (repository,)
|
+ (repository,)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -49,6 +49,7 @@ def prune_archives(
|
||||||
'''
|
'''
|
||||||
umask = storage_config.get('umask', None)
|
umask = storage_config.get('umask', None)
|
||||||
lock_wait = storage_config.get('lock_wait', None)
|
lock_wait = storage_config.get('lock_wait', None)
|
||||||
|
extra_borg_options = storage_config.get('extra_borg_options', {}).get('prune', '')
|
||||||
|
|
||||||
full_command = (
|
full_command = (
|
||||||
(local_path, 'prune')
|
(local_path, 'prune')
|
||||||
|
@ -61,6 +62,7 @@ def prune_archives(
|
||||||
+ (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
+ (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||||
+ (('--dry-run',) if dry_run else ())
|
+ (('--dry-run',) if dry_run else ())
|
||||||
+ (('--stats',) if stats else ())
|
+ (('--stats',) if stats else ())
|
||||||
|
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
|
||||||
+ (repository,)
|
+ (repository,)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -189,6 +189,7 @@ def run_actions(
|
||||||
logger.info('{}: Initializing repository'.format(repository))
|
logger.info('{}: Initializing repository'.format(repository))
|
||||||
borg_init.initialize_repository(
|
borg_init.initialize_repository(
|
||||||
repository,
|
repository,
|
||||||
|
storage,
|
||||||
arguments['init'].encryption_mode,
|
arguments['init'].encryption_mode,
|
||||||
arguments['init'].append_only,
|
arguments['init'].append_only,
|
||||||
arguments['init'].storage_quota,
|
arguments['init'].storage_quota,
|
||||||
|
|
|
@ -245,6 +245,29 @@ map:
|
||||||
Bypass Borg error about a previously unknown unencrypted repository. Defaults to
|
Bypass Borg error about a previously unknown unencrypted repository. Defaults to
|
||||||
false.
|
false.
|
||||||
example: true
|
example: true
|
||||||
|
extra_borg_options:
|
||||||
|
map:
|
||||||
|
init:
|
||||||
|
type: str
|
||||||
|
desc: Extra command-line options to pass to "borg init".
|
||||||
|
example: "--make-parent-dirs"
|
||||||
|
prune:
|
||||||
|
type: str
|
||||||
|
desc: Extra command-line options to pass to "borg prune".
|
||||||
|
example: "--save-space"
|
||||||
|
create:
|
||||||
|
type: str
|
||||||
|
desc: Extra command-line options to pass to "borg create".
|
||||||
|
example: "--no-files-cache"
|
||||||
|
check:
|
||||||
|
type: str
|
||||||
|
desc: Extra command-line options to pass to "borg check".
|
||||||
|
example: "--save-space"
|
||||||
|
desc: |
|
||||||
|
Additional options to pass directly to particular Borg commands, handy for Borg
|
||||||
|
options that borgmatic does not yet support natively. Note that borgmatic does
|
||||||
|
not perform any validation on these options. Running borgmatic with
|
||||||
|
"--verbosity 2" shows the exact Borg command-line invocation.
|
||||||
retention:
|
retention:
|
||||||
desc: |
|
desc: |
|
||||||
Retention policy for how many backups to keep in each category. See
|
Retention policy for how many backups to keep in each category. See
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -1,6 +1,6 @@
|
||||||
from setuptools import find_packages, setup
|
from setuptools import find_packages, setup
|
||||||
|
|
||||||
VERSION = '1.4.16'
|
VERSION = '1.4.17'
|
||||||
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
|
|
|
@ -296,3 +296,17 @@ def test_check_archives_with_retention_prefix():
|
||||||
module.check_archives(
|
module.check_archives(
|
||||||
repository='repo', storage_config={}, consistency_config=consistency_config
|
repository='repo', storage_config={}, consistency_config=consistency_config
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options():
|
||||||
|
checks = ('repository',)
|
||||||
|
consistency_config = {'check_last': None}
|
||||||
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
|
flexmock(module).should_receive('_make_check_flags').and_return(())
|
||||||
|
insert_execute_command_mock(('borg', 'check', '--extra', '--options', 'repo'))
|
||||||
|
|
||||||
|
module.check_archives(
|
||||||
|
repository='repo',
|
||||||
|
storage_config={'extra_borg_options': {'check': '--extra --options'}},
|
||||||
|
consistency_config=consistency_config,
|
||||||
|
)
|
||||||
|
|
|
@ -1092,3 +1092,28 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
|
||||||
},
|
},
|
||||||
storage_config={'archive_name_format': 'Documents_{hostname}-{now}'},
|
storage_config={'archive_name_format': 'Documents_{hostname}-{now}'},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
|
||||||
|
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||||
|
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||||
|
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||||
|
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||||
|
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||||
|
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||||
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
|
('borg', 'create', '--extra', '--options') + ARCHIVE_WITH_PATHS,
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
error_on_warnings=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
module.create_archive(
|
||||||
|
dry_run=False,
|
||||||
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={'extra_borg_options': {'create': '--extra --options'}},
|
||||||
|
)
|
||||||
|
|
|
@ -32,7 +32,7 @@ def test_initialize_repository_calls_borg_with_parameters():
|
||||||
insert_info_command_not_found_mock()
|
insert_info_command_not_found_mock()
|
||||||
insert_init_command_mock(INIT_COMMAND + ('repo',))
|
insert_init_command_mock(INIT_COMMAND + ('repo',))
|
||||||
|
|
||||||
module.initialize_repository(repository='repo', encryption_mode='repokey')
|
module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
|
||||||
|
|
||||||
|
|
||||||
def test_initialize_repository_raises_for_borg_init_error():
|
def test_initialize_repository_raises_for_borg_init_error():
|
||||||
|
@ -42,14 +42,16 @@ def test_initialize_repository_raises_for_borg_init_error():
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(subprocess.CalledProcessError):
|
with pytest.raises(subprocess.CalledProcessError):
|
||||||
module.initialize_repository(repository='repo', encryption_mode='repokey')
|
module.initialize_repository(
|
||||||
|
repository='repo', storage_config={}, encryption_mode='repokey'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_initialize_repository_skips_initialization_when_repository_already_exists():
|
def test_initialize_repository_skips_initialization_when_repository_already_exists():
|
||||||
insert_info_command_found_mock()
|
insert_info_command_found_mock()
|
||||||
flexmock(module).should_receive('execute_command_without_capture').never()
|
flexmock(module).should_receive('execute_command_without_capture').never()
|
||||||
|
|
||||||
module.initialize_repository(repository='repo', encryption_mode='repokey')
|
module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
|
||||||
|
|
||||||
|
|
||||||
def test_initialize_repository_raises_for_unknown_info_command_error():
|
def test_initialize_repository_raises_for_unknown_info_command_error():
|
||||||
|
@ -58,21 +60,27 @@ def test_initialize_repository_raises_for_unknown_info_command_error():
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(subprocess.CalledProcessError):
|
with pytest.raises(subprocess.CalledProcessError):
|
||||||
module.initialize_repository(repository='repo', encryption_mode='repokey')
|
module.initialize_repository(
|
||||||
|
repository='repo', storage_config={}, encryption_mode='repokey'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_initialize_repository_with_append_only_calls_borg_with_append_only_parameter():
|
def test_initialize_repository_with_append_only_calls_borg_with_append_only_parameter():
|
||||||
insert_info_command_not_found_mock()
|
insert_info_command_not_found_mock()
|
||||||
insert_init_command_mock(INIT_COMMAND + ('--append-only', 'repo'))
|
insert_init_command_mock(INIT_COMMAND + ('--append-only', 'repo'))
|
||||||
|
|
||||||
module.initialize_repository(repository='repo', encryption_mode='repokey', append_only=True)
|
module.initialize_repository(
|
||||||
|
repository='repo', storage_config={}, encryption_mode='repokey', append_only=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_initialize_repository_with_storage_quota_calls_borg_with_storage_quota_parameter():
|
def test_initialize_repository_with_storage_quota_calls_borg_with_storage_quota_parameter():
|
||||||
insert_info_command_not_found_mock()
|
insert_info_command_not_found_mock()
|
||||||
insert_init_command_mock(INIT_COMMAND + ('--storage-quota', '5G', 'repo'))
|
insert_init_command_mock(INIT_COMMAND + ('--storage-quota', '5G', 'repo'))
|
||||||
|
|
||||||
module.initialize_repository(repository='repo', encryption_mode='repokey', storage_quota='5G')
|
module.initialize_repository(
|
||||||
|
repository='repo', storage_config={}, encryption_mode='repokey', storage_quota='5G'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
|
def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
|
||||||
|
@ -80,7 +88,7 @@ def test_initialize_repository_with_log_info_calls_borg_with_info_parameter():
|
||||||
insert_init_command_mock(INIT_COMMAND + ('--info', 'repo'))
|
insert_init_command_mock(INIT_COMMAND + ('--info', 'repo'))
|
||||||
insert_logging_mock(logging.INFO)
|
insert_logging_mock(logging.INFO)
|
||||||
|
|
||||||
module.initialize_repository(repository='repo', encryption_mode='repokey')
|
module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
|
||||||
|
|
||||||
|
|
||||||
def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
|
def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
|
||||||
|
@ -88,18 +96,33 @@ def test_initialize_repository_with_log_debug_calls_borg_with_debug_parameter():
|
||||||
insert_init_command_mock(INIT_COMMAND + ('--debug', 'repo'))
|
insert_init_command_mock(INIT_COMMAND + ('--debug', 'repo'))
|
||||||
insert_logging_mock(logging.DEBUG)
|
insert_logging_mock(logging.DEBUG)
|
||||||
|
|
||||||
module.initialize_repository(repository='repo', encryption_mode='repokey')
|
module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
|
||||||
|
|
||||||
|
|
||||||
def test_initialize_repository_with_local_path_calls_borg_via_local_path():
|
def test_initialize_repository_with_local_path_calls_borg_via_local_path():
|
||||||
insert_info_command_not_found_mock()
|
insert_info_command_not_found_mock()
|
||||||
insert_init_command_mock(('borg1',) + INIT_COMMAND[1:] + ('repo',))
|
insert_init_command_mock(('borg1',) + INIT_COMMAND[1:] + ('repo',))
|
||||||
|
|
||||||
module.initialize_repository(repository='repo', encryption_mode='repokey', local_path='borg1')
|
module.initialize_repository(
|
||||||
|
repository='repo', storage_config={}, encryption_mode='repokey', local_path='borg1'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_initialize_repository_with_remote_path_calls_borg_with_remote_path_parameter():
|
def test_initialize_repository_with_remote_path_calls_borg_with_remote_path_parameter():
|
||||||
insert_info_command_not_found_mock()
|
insert_info_command_not_found_mock()
|
||||||
insert_init_command_mock(INIT_COMMAND + ('--remote-path', 'borg1', 'repo'))
|
insert_init_command_mock(INIT_COMMAND + ('--remote-path', 'borg1', 'repo'))
|
||||||
|
|
||||||
module.initialize_repository(repository='repo', encryption_mode='repokey', remote_path='borg1')
|
module.initialize_repository(
|
||||||
|
repository='repo', storage_config={}, encryption_mode='repokey', remote_path='borg1'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_initialize_repository_with_extra_borg_options_calls_borg_with_extra_options():
|
||||||
|
insert_info_command_not_found_mock()
|
||||||
|
insert_init_command_mock(INIT_COMMAND + ('--extra', '--options', 'repo'))
|
||||||
|
|
||||||
|
module.initialize_repository(
|
||||||
|
repository='repo',
|
||||||
|
storage_config={'extra_borg_options': {'init': '--extra --options'}},
|
||||||
|
encryption_mode='repokey',
|
||||||
|
)
|
||||||
|
|
|
@ -188,3 +188,18 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
|
||||||
storage_config=storage_config,
|
storage_config=storage_config,
|
||||||
retention_config=retention_config,
|
retention_config=retention_config,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options():
|
||||||
|
retention_config = flexmock()
|
||||||
|
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
|
||||||
|
BASE_PRUNE_FLAGS
|
||||||
|
)
|
||||||
|
insert_execute_command_mock(PRUNE_COMMAND + ('--extra', '--options', 'repo'), logging.INFO)
|
||||||
|
|
||||||
|
module.prune_archives(
|
||||||
|
dry_run=False,
|
||||||
|
repository='repo',
|
||||||
|
storage_config={'extra_borg_options': {'prune': '--extra --options'}},
|
||||||
|
retention_config=retention_config,
|
||||||
|
)
|
||||||
|
|
Loading…
Reference in a new issue