48: Add "local_path" to configuration for specifying an alternative Borg executable path.

This commit is contained in:
Dan Helfman 2018-01-14 16:35:24 -08:00
parent b8f6bab12d
commit cd189c4fe4
11 changed files with 119 additions and 26 deletions

1
NEWS
View file

@ -1,6 +1,7 @@
1.1.13.dev0 1.1.13.dev0
* #54: Fix for incorrect consistency check flags passed to Borg when all three checks ("repository", * #54: Fix for incorrect consistency check flags passed to Borg when all three checks ("repository",
"archives", and "extract") are specified in borgmatic configuration. "archives", and "extract") are specified in borgmatic configuration.
* #48: Add "local_path" to configuration for specifying an alternative Borg executable path.
* #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed * #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed
includes/excludes. includes/excludes.
* Moved issue tracker from Taiga to integrated Gitea tracker at * Moved issue tracker from Taiga to integrated Gitea tracker at

View file

@ -60,10 +60,10 @@ def _make_check_flags(checks, check_last=None):
) + last_flag ) + last_flag
def check_archives(verbosity, repository, consistency_config, remote_path=None): def check_archives(verbosity, repository, consistency_config, local_path='borg', remote_path=None):
''' '''
Given a verbosity flag, a local or remote repository path, a consistency config dict, and a Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
command to run, check the contained Borg archives for consistency. local/remote commands to run, check the contained Borg archives for consistency.
If there are no consistency checks to run, skip running them. If there are no consistency checks to run, skip running them.
''' '''
@ -78,7 +78,7 @@ def check_archives(verbosity, repository, consistency_config, remote_path=None):
}.get(verbosity, ()) }.get(verbosity, ())
full_command = ( full_command = (
'borg', 'check', local_path, 'check',
repository, repository,
) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
@ -89,4 +89,4 @@ def check_archives(verbosity, repository, consistency_config, remote_path=None):
subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
if 'extract' in checks: if 'extract' in checks:
extract.extract_last_archive_dry_run(verbosity, repository, remote_path) extract.extract_last_archive_dry_run(verbosity, repository, local_path, remote_path)

View file

@ -85,7 +85,7 @@ def _make_exclude_flags(location_config, exclude_filename=None):
def create_archive( def create_archive(
verbosity, repository, location_config, storage_config, verbosity, repository, location_config, storage_config, local_path='borg', remote_path=None,
): ):
''' '''
Given a vebosity flag, a local or remote repository path, a location config dict, and a storage Given a vebosity flag, a local or remote repository path, a location config dict, and a storage
@ -117,7 +117,6 @@ def create_archive(
one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else () one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
files_cache = location_config.get('files_cache') files_cache = location_config.get('files_cache')
files_cache_flags = ('--files-cache', files_cache) if files_cache else () files_cache_flags = ('--files-cache', files_cache) if files_cache else ()
remote_path = location_config.get('remote_path')
remote_path_flags = ('--remote-path', remote_path) if remote_path else () remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
verbosity_flags = { verbosity_flags = {
VERBOSITY_SOME: ('--info', '--stats',), VERBOSITY_SOME: ('--info', '--stats',),
@ -127,7 +126,7 @@ def create_archive(
archive_name_format = storage_config.get('archive_name_format', default_archive_name_format) archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
full_command = ( full_command = (
'borg', 'create', local_path, 'create',
'{repository}::{archive_name_format}'.format( '{repository}::{archive_name_format}'.format(
repository=repository, repository=repository,
archive_name_format=archive_name_format, archive_name_format=archive_name_format,

View file

@ -8,7 +8,7 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def extract_last_archive_dry_run(verbosity, repository, remote_path=None): def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remote_path=None):
''' '''
Perform an extraction dry-run of just the most recent archive. If there are no archives, skip Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
the dry-run. the dry-run.
@ -20,7 +20,7 @@ def extract_last_archive_dry_run(verbosity, repository, remote_path=None):
}.get(verbosity, ()) }.get(verbosity, ())
full_list_command = ( full_list_command = (
'borg', 'list', local_path, 'list',
'--short', '--short',
repository, repository,
) + remote_path_flags + verbosity_flags ) + remote_path_flags + verbosity_flags
@ -33,7 +33,7 @@ def extract_last_archive_dry_run(verbosity, repository, remote_path=None):
list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else () list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else ()
full_extract_command = ( full_extract_command = (
'borg', 'extract', local_path, 'extract',
'--dry-run', '--dry-run',
'{repository}::{last_archive_name}'.format( '{repository}::{last_archive_name}'.format(
repository=repository, repository=repository,

View file

@ -32,7 +32,7 @@ def _make_prune_flags(retention_config):
) )
def prune_archives(verbosity, repository, retention_config, remote_path=None): def prune_archives(verbosity, repository, retention_config, local_path='borg', remote_path=None):
''' '''
Given a verbosity flag, a local or remote repository path, a retention config dict, prune Borg Given a verbosity flag, a local or remote repository path, a retention config dict, prune Borg
archives according the the retention policy specified in that configuration. archives according the the retention policy specified in that configuration.
@ -44,7 +44,7 @@ def prune_archives(verbosity, repository, retention_config, remote_path=None):
}.get(verbosity, ()) }.get(verbosity, ())
full_command = ( full_command = (
'borg', 'prune', local_path, 'prune',
repository, repository,
) + tuple( ) + tuple(
element element

View file

@ -93,6 +93,7 @@ def run_configuration(config_filename, args): # pragma: no cover
) )
try: try:
local_path = location.get('local_path', 'borg')
remote_path = location.get('remote_path') remote_path = location.get('remote_path')
create.initialize_environment(storage) create.initialize_environment(storage)
hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup') hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup')
@ -101,7 +102,13 @@ def run_configuration(config_filename, args): # pragma: no cover
repository = os.path.expanduser(unexpanded_repository) repository = os.path.expanduser(unexpanded_repository)
if args.prune: if args.prune:
logger.info('{}: Pruning archives'.format(repository)) logger.info('{}: Pruning archives'.format(repository))
prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) prune.prune_archives(
args.verbosity,
repository,
retention,
local_path=local_path,
remote_path=remote_path,
)
if args.create: if args.create:
logger.info('{}: Creating archive'.format(repository)) logger.info('{}: Creating archive'.format(repository))
create.create_archive( create.create_archive(
@ -109,10 +116,18 @@ def run_configuration(config_filename, args): # pragma: no cover
repository, repository,
location, location,
storage, storage,
local_path=local_path,
remote_path=remote_path,
) )
if args.check: if args.check:
logger.info('{}: Running consistency checks'.format(repository)) logger.info('{}: Running consistency checks'.format(repository))
check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) check.check_archives(
args.verbosity,
repository,
consistency,
local_path=local_path,
remote_path=remote_path,
)
hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup') hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup')
except (OSError, CalledProcessError): except (OSError, CalledProcessError):

View file

@ -29,6 +29,10 @@ map:
https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
details. details.
example: ctime,size,inode example: ctime,size,inode
local_path:
type: scalar
desc: Alternate Borg local executable. Defaults to "borg".
example: borg1
remote_path: remote_path:
type: scalar type: scalar
desc: Alternate Borg remote executable. Defaults to "borg". desc: Alternate Borg remote executable. Defaults to "borg".

View file

@ -87,7 +87,7 @@ def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
('repository', 'archives', 'other'), ('repository', 'archives', 'other'),
), ),
) )
def test_check_archives_should_call_borg_with_parameters(checks): def test_check_archives_calls_borg_with_parameters(checks):
check_last = flexmock() check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock consistency_config = flexmock().should_receive('get').and_return(check_last).mock
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
@ -107,7 +107,7 @@ def test_check_archives_should_call_borg_with_parameters(checks):
) )
def test_check_archives_with_extract_check_should_call_extract_only(): def test_check_archives_with_extract_check_calls_extract_only():
checks = ('extract',) checks = ('extract',)
check_last = flexmock() check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock consistency_config = flexmock().should_receive('get').and_return(check_last).mock
@ -123,7 +123,7 @@ def test_check_archives_with_extract_check_should_call_extract_only():
) )
def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter(): def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
checks = ('repository',) checks = ('repository',)
consistency_config = flexmock().should_receive('get').and_return(None).mock consistency_config = flexmock().should_receive('get').and_return(None).mock
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
@ -140,7 +140,7 @@ def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter
) )
def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter(): def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
checks = ('repository',) checks = ('repository',)
consistency_config = flexmock().should_receive('get').and_return(None).mock consistency_config = flexmock().should_receive('get').and_return(None).mock
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
@ -157,7 +157,7 @@ def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_paramete
) )
def test_check_archives_without_any_checks_should_bail(): def test_check_archives_without_any_checks_bails():
consistency_config = flexmock().should_receive('get').and_return(None).mock consistency_config = flexmock().should_receive('get').and_return(None).mock
flexmock(module).should_receive('_parse_checks').and_return(()) flexmock(module).should_receive('_parse_checks').and_return(())
insert_subprocess_never() insert_subprocess_never()
@ -169,7 +169,29 @@ def test_check_archives_without_any_checks_should_bail():
) )
def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters(): def test_check_archives_with_local_path_calls_borg_via_local_path():
checks = ('repository',)
check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock()
insert_subprocess_mock(
('borg1', 'check', 'repo'),
stdout=stdout, stderr=STDOUT,
)
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull')
module.check_archives(
verbosity=None,
repository='repo',
consistency_config=consistency_config,
local_path='borg1',
)
def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters():
checks = ('repository',) checks = ('repository',)
check_last = flexmock() check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock consistency_config = flexmock().should_receive('get').and_return(check_last).mock

View file

@ -357,6 +357,26 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
) )
def test_create_archive_with_local_path_calls_borg_via_local_path():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(('borg1',) + CREATE_COMMAND[1:])
module.create_archive(
verbosity=None,
repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={},
local_path='borg1',
)
def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(): def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -370,10 +390,10 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
location_config={ location_config={
'source_directories': ['foo', 'bar'], 'source_directories': ['foo', 'bar'],
'repositories': ['repo'], 'repositories': ['repo'],
'remote_path': 'borg1',
'exclude_patterns': None, 'exclude_patterns': None,
}, },
storage_config={}, storage_config={},
remote_path='borg1',
) )

View file

@ -83,6 +83,23 @@ def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_
) )
def test_extract_last_archive_dry_run_should_call_borg_via_local_path():
flexmock(sys.stdout).encoding = 'utf-8'
insert_subprocess_check_output_mock(
('borg1', 'list', '--short', 'repo'),
result='archive1\narchive2\n'.encode('utf-8'),
)
insert_subprocess_mock(
('borg1', 'extract', '--dry-run', 'repo::archive2'),
)
module.extract_last_archive_dry_run(
verbosity=None,
repository='repo',
local_path='borg1',
)
def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters(): def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters():
flexmock(sys.stdout).encoding = 'utf-8' flexmock(sys.stdout).encoding = 'utf-8'
insert_subprocess_check_output_mock( insert_subprocess_check_output_mock(

View file

@ -18,7 +18,7 @@ BASE_PRUNE_FLAGS = (
) )
def test_make_prune_flags_should_return_flags_from_config_plus_default_prefix(): def test_make_prune_flags_returns_flags_from_config_plus_default_prefix():
retention_config = OrderedDict( retention_config = OrderedDict(
( (
('keep_daily', 1), ('keep_daily', 1),
@ -55,7 +55,7 @@ PRUNE_COMMAND = (
) )
def test_prune_archives_should_call_borg_with_parameters(): def test_prune_archives_calls_borg_with_parameters():
retention_config = flexmock() retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS, BASE_PRUNE_FLAGS,
@ -69,7 +69,7 @@ def test_prune_archives_should_call_borg_with_parameters():
) )
def test_prune_archives_with_verbosity_some_should_call_borg_with_info_parameter(): def test_prune_archives_with_verbosity_some_calls_borg_with_info_parameter():
retention_config = flexmock() retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS, BASE_PRUNE_FLAGS,
@ -83,7 +83,7 @@ def test_prune_archives_with_verbosity_some_should_call_borg_with_info_parameter
) )
def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_parameter(): def test_prune_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
retention_config = flexmock() retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS, BASE_PRUNE_FLAGS,
@ -97,7 +97,22 @@ def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_paramete
) )
def test_prune_archives_with_remote_path_should_call_borg_with_remote_path_parameters(): def test_prune_archives_with_local_path_calls_borg_via_local_path():
retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS,
)
insert_subprocess_mock(('borg1',) + PRUNE_COMMAND[1:])
module.prune_archives(
verbosity=None,
repository='repo',
retention_config=retention_config,
local_path='borg1',
)
def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters():
retention_config = flexmock() retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS, BASE_PRUNE_FLAGS,