Support for Borg --lock-wait option for the maximum wait for a repository/cache lock (#56).

This commit is contained in:
Dan Helfman 2018-02-19 15:51:04 -08:00
parent a87036ee46
commit 2d3f5fa05d
12 changed files with 132 additions and 16 deletions

6
NEWS
View file

@ -1,7 +1,9 @@
1.1.15.dev0 1.1.15
* Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file. * Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file.
* Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together. * Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together.
* #55: Fix for missing tags/releases from Gitea and GitHub project hosting. Work-around for behavior introduced in Borg 1.1.3: https://github.com/borgbackup/borg/issues/3298
* #55: Fix for missing tags/releases on Gitea and GitHub project hosting.
* #56: Support for Borg --lock-wait option for the maximum wait for a repository/cache lock.
* #58: Support for using tilde in exclude_patterns to reference home directory. * #58: Support for using tilde in exclude_patterns to reference home directory.
1.1.14 1.1.14

View file

@ -60,18 +60,23 @@ def _make_check_flags(checks, check_last=None):
) + last_flag ) + last_flag
def check_archives(verbosity, repository, consistency_config, local_path='borg', remote_path=None): def check_archives(verbosity, repository, storage_config, consistency_config, local_path='borg',
remote_path=None):
''' '''
Given a verbosity flag, a local or remote repository path, a consistency config dict, and a Given a verbosity flag, a local or remote repository path, a storage config dict, a consistency
local/remote commands to run, check the contained Borg archives for consistency. config dict, and a local/remote commands to run, check the contained Borg archives for
consistency.
If there are no consistency checks to run, skip running them. If there are no consistency checks to run, skip running them.
''' '''
checks = _parse_checks(consistency_config) checks = _parse_checks(consistency_config)
check_last = consistency_config.get('check_last', None) check_last = consistency_config.get('check_last', None)
lock_wait = None
if set(checks).intersection(set(DEFAULT_CHECKS)): if set(checks).intersection(set(DEFAULT_CHECKS)):
remote_path_flags = ('--remote-path', remote_path) if remote_path else () remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
lock_wait = storage_config.get('lock_wait', None)
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
verbosity_flags = { verbosity_flags = {
VERBOSITY_SOME: ('--info',), VERBOSITY_SOME: ('--info',),
VERBOSITY_LOTS: ('--debug',), VERBOSITY_LOTS: ('--debug',),
@ -80,7 +85,7 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg',
full_command = ( full_command = (
local_path, 'check', local_path, 'check',
repository, repository,
) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags ) + _make_check_flags(checks, check_last) + remote_path_flags + lock_wait_flags + verbosity_flags
# The check command spews to stdout/stderr even without the verbose flag. Suppress it. # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
stdout = None if verbosity_flags else open(os.devnull, 'w') stdout = None if verbosity_flags else open(os.devnull, 'w')
@ -89,4 +94,4 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg',
subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
if 'extract' in checks: if 'extract' in checks:
extract.extract_last_archive_dry_run(verbosity, repository, local_path, remote_path) extract.extract_last_archive_dry_run(verbosity, repository, lock_wait, local_path, remote_path)

View file

@ -129,6 +129,8 @@ def create_archive(
remote_rate_limit_flags = ('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else () remote_rate_limit_flags = ('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ()
umask = storage_config.get('umask', None) umask = storage_config.get('umask', None)
umask_flags = ('--umask', str(umask)) if umask else () umask_flags = ('--umask', str(umask)) if umask else ()
lock_wait = storage_config.get('lock_wait', None)
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else () one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
files_cache = location_config.get('files_cache') files_cache = location_config.get('files_cache')
files_cache_flags = ('--files-cache', files_cache) if files_cache else () files_cache_flags = ('--files-cache', files_cache) if files_cache else ()
@ -149,7 +151,7 @@ def create_archive(
), ),
) + sources + pattern_flags + exclude_flags + compression_flags + remote_rate_limit_flags + \ ) + sources + pattern_flags + exclude_flags + compression_flags + remote_rate_limit_flags + \
one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \ one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \
verbosity_flags + dry_run_flags lock_wait_flags + verbosity_flags + dry_run_flags
logger.debug(' '.join(full_command)) logger.debug(' '.join(full_command))
subprocess.check_call(full_command) subprocess.check_call(full_command)

View file

@ -8,12 +8,13 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remote_path=None): def extract_last_archive_dry_run(verbosity, repository, lock_wait=None, local_path='borg', remote_path=None):
''' '''
Perform an extraction dry-run of just the most recent archive. If there are no archives, skip Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
the dry-run. the dry-run.
''' '''
remote_path_flags = ('--remote-path', remote_path) if remote_path else () remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
verbosity_flags = { verbosity_flags = {
VERBOSITY_SOME: ('--info',), VERBOSITY_SOME: ('--info',),
VERBOSITY_LOTS: ('--debug',), VERBOSITY_LOTS: ('--debug',),
@ -23,7 +24,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot
local_path, 'list', local_path, 'list',
'--short', '--short',
repository, repository,
) + remote_path_flags + verbosity_flags ) + remote_path_flags + lock_wait_flags + verbosity_flags
list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding) list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
@ -39,7 +40,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot
repository=repository, repository=repository,
last_archive_name=last_archive_name, last_archive_name=last_archive_name,
), ),
) + remote_path_flags + verbosity_flags + list_flag ) + remote_path_flags + lock_wait_flags + verbosity_flags + list_flag
logger.debug(' '.join(full_extract_command)) logger.debug(' '.join(full_extract_command))
subprocess.check_call(full_extract_command) subprocess.check_call(full_extract_command)

View file

@ -32,12 +32,16 @@ def _make_prune_flags(retention_config):
) )
def prune_archives(verbosity, dry_run, repository, retention_config, local_path='borg', remote_path=None): def prune_archives(verbosity, dry_run, repository, storage_config, retention_config,
local_path='borg', remote_path=None):
''' '''
Given verbosity/dry-run flags, a local or remote repository path, a retention config dict, prune Given verbosity/dry-run flags, a local or remote repository path, a storage config dict, and a
Borg archives according the the retention policy specified in that configuration. retention config dict, prune Borg archives according the the retention policy specified in that
configuration.
''' '''
remote_path_flags = ('--remote-path', remote_path) if remote_path else () remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
lock_wait = storage_config.get('lock_wait', None)
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
verbosity_flags = { verbosity_flags = {
VERBOSITY_SOME: ('--info', '--stats',), VERBOSITY_SOME: ('--info', '--stats',),
VERBOSITY_LOTS: ('--debug', '--stats', '--list'), VERBOSITY_LOTS: ('--debug', '--stats', '--list'),
@ -51,7 +55,7 @@ def prune_archives(verbosity, dry_run, repository, retention_config, local_path=
element element
for pair in _make_prune_flags(retention_config) for pair in _make_prune_flags(retention_config)
for element in pair for element in pair
) + remote_path_flags + verbosity_flags + dry_run_flags ) + remote_path_flags + lock_wait_flags + verbosity_flags + dry_run_flags
logger.debug(' '.join(full_command)) logger.debug(' '.join(full_command))
subprocess.check_call(full_command) subprocess.check_call(full_command)

View file

@ -113,6 +113,7 @@ def run_configuration(config_filename, args): # pragma: no cover
args.verbosity, args.verbosity,
args.dry_run, args.dry_run,
repository, repository,
storage,
retention, retention,
local_path=local_path, local_path=local_path,
remote_path=remote_path, remote_path=remote_path,
@ -133,6 +134,7 @@ def run_configuration(config_filename, args): # pragma: no cover
check.check_archives( check.check_archives(
args.verbosity, args.verbosity,
repository, repository,
storage,
consistency, consistency,
local_path=local_path, local_path=local_path,
remote_path=remote_path, remote_path=remote_path,

View file

@ -139,6 +139,10 @@ map:
type: scalar type: scalar
desc: Umask to be used for borg create. desc: Umask to be used for borg create.
example: 0077 example: 0077
lock_wait:
type: int
desc: Maximum seconds to wait for acquiring a repository/cache lock.
example: 5
archive_name_format: archive_name_format:
type: scalar type: scalar
desc: | desc: |
@ -152,6 +156,7 @@ map:
desc: | desc: |
Retention policy for how many backups to keep in each category. See Retention policy for how many backups to keep in each category. See
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details. https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
At least one of the "keep" options is required for pruning to work.
map: map:
keep_within: keep_within:
type: scalar type: scalar

View file

@ -103,6 +103,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
module.check_archives( module.check_archives(
verbosity=None, verbosity=None,
repository='repo', repository='repo',
storage_config={},
consistency_config=consistency_config, consistency_config=consistency_config,
) )
@ -119,6 +120,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
module.check_archives( module.check_archives(
verbosity=None, verbosity=None,
repository='repo', repository='repo',
storage_config={},
consistency_config=consistency_config, consistency_config=consistency_config,
) )
@ -136,6 +138,7 @@ def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
module.check_archives( module.check_archives(
verbosity=VERBOSITY_SOME, verbosity=VERBOSITY_SOME,
repository='repo', repository='repo',
storage_config={},
consistency_config=consistency_config, consistency_config=consistency_config,
) )
@ -153,6 +156,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
module.check_archives( module.check_archives(
verbosity=VERBOSITY_LOTS, verbosity=VERBOSITY_LOTS,
repository='repo', repository='repo',
storage_config={},
consistency_config=consistency_config, consistency_config=consistency_config,
) )
@ -165,6 +169,7 @@ def test_check_archives_without_any_checks_bails():
module.check_archives( module.check_archives(
verbosity=None, verbosity=None,
repository='repo', repository='repo',
storage_config={},
consistency_config=consistency_config, consistency_config=consistency_config,
) )
@ -186,6 +191,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
module.check_archives( module.check_archives(
verbosity=None, verbosity=None,
repository='repo', repository='repo',
storage_config={},
consistency_config=consistency_config, consistency_config=consistency_config,
local_path='borg1', local_path='borg1',
) )
@ -208,6 +214,29 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
module.check_archives( module.check_archives(
verbosity=None, verbosity=None,
repository='repo', repository='repo',
storage_config={},
consistency_config=consistency_config, consistency_config=consistency_config,
remote_path='borg1', remote_path='borg1',
) )
def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
checks = ('repository',)
check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock()
insert_subprocess_mock(
('borg', 'check', 'repo', '--lock-wait', '5'),
stdout=stdout, stderr=STDOUT,
)
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull')
module.check_archives(
verbosity=None,
repository='repo',
storage_config={'lock_wait': 5},
consistency_config=consistency_config,
)

View file

@ -518,6 +518,26 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
) )
def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + ('--lock-wait', '5'))
module.create_archive(
verbosity=None,
dry_run=False,
repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={'lock_wait': 5},
)
def test_create_archive_with_source_directories_glob_expands(): def test_create_archive_with_source_directories_glob_expands():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(()) flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_write_pattern_file').and_return(None)

View file

@ -34,6 +34,7 @@ def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
module.extract_last_archive_dry_run( module.extract_last_archive_dry_run(
verbosity=None, verbosity=None,
repository='repo', repository='repo',
lock_wait=None,
) )
@ -48,6 +49,7 @@ def test_extract_last_archive_dry_run_without_any_archives_should_bail():
module.extract_last_archive_dry_run( module.extract_last_archive_dry_run(
verbosity=None, verbosity=None,
repository='repo', repository='repo',
lock_wait=None,
) )
@ -64,6 +66,7 @@ def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_
module.extract_last_archive_dry_run( module.extract_last_archive_dry_run(
verbosity=VERBOSITY_SOME, verbosity=VERBOSITY_SOME,
repository='repo', repository='repo',
lock_wait=None,
) )
@ -80,6 +83,7 @@ def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_
module.extract_last_archive_dry_run( module.extract_last_archive_dry_run(
verbosity=VERBOSITY_LOTS, verbosity=VERBOSITY_LOTS,
repository='repo', repository='repo',
lock_wait=None,
) )
@ -96,6 +100,7 @@ def test_extract_last_archive_dry_run_should_call_borg_via_local_path():
module.extract_last_archive_dry_run( module.extract_last_archive_dry_run(
verbosity=None, verbosity=None,
repository='repo', repository='repo',
lock_wait=None,
local_path='borg1', local_path='borg1',
) )
@ -113,5 +118,23 @@ def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_paramete
module.extract_last_archive_dry_run( module.extract_last_archive_dry_run(
verbosity=None, verbosity=None,
repository='repo', repository='repo',
lock_wait=None,
remote_path='borg1', remote_path='borg1',
) )
def test_extract_last_archive_dry_run_should_call_borg_with_lock_wait_parameters():
flexmock(sys.stdout).encoding = 'utf-8'
insert_subprocess_check_output_mock(
('borg', 'list', '--short', 'repo', '--lock-wait', '5'),
result='archive1\narchive2\n'.encode('utf-8'),
)
insert_subprocess_mock(
('borg', 'extract', '--dry-run', 'repo::archive2', '--lock-wait', '5'),
)
module.extract_last_archive_dry_run(
verbosity=None,
repository='repo',
lock_wait=5,
)

View file

@ -66,6 +66,7 @@ def test_prune_archives_calls_borg_with_parameters():
verbosity=None, verbosity=None,
dry_run=False, dry_run=False,
repository='repo', repository='repo',
storage_config={},
retention_config=retention_config, retention_config=retention_config,
) )
@ -79,6 +80,7 @@ def test_prune_archives_with_verbosity_some_calls_borg_with_info_parameter():
module.prune_archives( module.prune_archives(
repository='repo', repository='repo',
storage_config={},
verbosity=VERBOSITY_SOME, verbosity=VERBOSITY_SOME,
dry_run=False, dry_run=False,
retention_config=retention_config, retention_config=retention_config,
@ -94,6 +96,7 @@ def test_prune_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
module.prune_archives( module.prune_archives(
repository='repo', repository='repo',
storage_config={},
verbosity=VERBOSITY_LOTS, verbosity=VERBOSITY_LOTS,
dry_run=False, dry_run=False,
retention_config=retention_config, retention_config=retention_config,
@ -109,6 +112,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter():
module.prune_archives( module.prune_archives(
repository='repo', repository='repo',
storage_config={},
verbosity=None, verbosity=None,
dry_run=True, dry_run=True,
retention_config=retention_config, retention_config=retention_config,
@ -126,6 +130,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path():
verbosity=None, verbosity=None,
dry_run=False, dry_run=False,
repository='repo', repository='repo',
storage_config={},
retention_config=retention_config, retention_config=retention_config,
local_path='borg1', local_path='borg1',
) )
@ -142,6 +147,24 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters(
verbosity=None, verbosity=None,
dry_run=False, dry_run=False,
repository='repo', repository='repo',
storage_config={},
retention_config=retention_config, retention_config=retention_config,
remote_path='borg1', remote_path='borg1',
) )
def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
storage_config = {'lock_wait': 5}
retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS,
)
insert_subprocess_mock(PRUNE_COMMAND + ('--lock-wait', '5'))
module.prune_archives(
verbosity=None,
dry_run=False,
repository='repo',
storage_config=storage_config,
retention_config=retention_config,
)

View file

@ -1,7 +1,7 @@
from setuptools import setup, find_packages from setuptools import setup, find_packages
VERSION = '1.1.15.dev' VERSION = '1.1.15'
setup( setup(