Support for Borg --lock-wait option for the maximum wait for a repository/cache lock (#56).
This commit is contained in:
parent
a87036ee46
commit
2d3f5fa05d
12 changed files with 132 additions and 16 deletions
6
NEWS
6
NEWS
|
@ -1,7 +1,9 @@
|
|||
1.1.15.dev0
|
||||
1.1.15
|
||||
* Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file.
|
||||
* Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together.
|
||||
* #55: Fix for missing tags/releases from Gitea and GitHub project hosting.
|
||||
Work-around for behavior introduced in Borg 1.1.3: https://github.com/borgbackup/borg/issues/3298
|
||||
* #55: Fix for missing tags/releases on Gitea and GitHub project hosting.
|
||||
* #56: Support for Borg --lock-wait option for the maximum wait for a repository/cache lock.
|
||||
* #58: Support for using tilde in exclude_patterns to reference home directory.
|
||||
|
||||
1.1.14
|
||||
|
|
|
@ -60,18 +60,23 @@ def _make_check_flags(checks, check_last=None):
|
|||
) + last_flag
|
||||
|
||||
|
||||
def check_archives(verbosity, repository, consistency_config, local_path='borg', remote_path=None):
|
||||
def check_archives(verbosity, repository, storage_config, consistency_config, local_path='borg',
|
||||
remote_path=None):
|
||||
'''
|
||||
Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
|
||||
local/remote commands to run, check the contained Borg archives for consistency.
|
||||
Given a verbosity flag, a local or remote repository path, a storage config dict, a consistency
|
||||
config dict, and a local/remote commands to run, check the contained Borg archives for
|
||||
consistency.
|
||||
|
||||
If there are no consistency checks to run, skip running them.
|
||||
'''
|
||||
checks = _parse_checks(consistency_config)
|
||||
check_last = consistency_config.get('check_last', None)
|
||||
lock_wait = None
|
||||
|
||||
if set(checks).intersection(set(DEFAULT_CHECKS)):
|
||||
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
||||
lock_wait = storage_config.get('lock_wait', None)
|
||||
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
|
||||
verbosity_flags = {
|
||||
VERBOSITY_SOME: ('--info',),
|
||||
VERBOSITY_LOTS: ('--debug',),
|
||||
|
@ -80,7 +85,7 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg',
|
|||
full_command = (
|
||||
local_path, 'check',
|
||||
repository,
|
||||
) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
|
||||
) + _make_check_flags(checks, check_last) + remote_path_flags + lock_wait_flags + verbosity_flags
|
||||
|
||||
# The check command spews to stdout/stderr even without the verbose flag. Suppress it.
|
||||
stdout = None if verbosity_flags else open(os.devnull, 'w')
|
||||
|
@ -89,4 +94,4 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg',
|
|||
subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
|
||||
|
||||
if 'extract' in checks:
|
||||
extract.extract_last_archive_dry_run(verbosity, repository, local_path, remote_path)
|
||||
extract.extract_last_archive_dry_run(verbosity, repository, lock_wait, local_path, remote_path)
|
||||
|
|
|
@ -129,6 +129,8 @@ def create_archive(
|
|||
remote_rate_limit_flags = ('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ()
|
||||
umask = storage_config.get('umask', None)
|
||||
umask_flags = ('--umask', str(umask)) if umask else ()
|
||||
lock_wait = storage_config.get('lock_wait', None)
|
||||
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
|
||||
one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
|
||||
files_cache = location_config.get('files_cache')
|
||||
files_cache_flags = ('--files-cache', files_cache) if files_cache else ()
|
||||
|
@ -149,7 +151,7 @@ def create_archive(
|
|||
),
|
||||
) + sources + pattern_flags + exclude_flags + compression_flags + remote_rate_limit_flags + \
|
||||
one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \
|
||||
verbosity_flags + dry_run_flags
|
||||
lock_wait_flags + verbosity_flags + dry_run_flags
|
||||
|
||||
logger.debug(' '.join(full_command))
|
||||
subprocess.check_call(full_command)
|
||||
|
|
|
@ -8,12 +8,13 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remote_path=None):
|
||||
def extract_last_archive_dry_run(verbosity, repository, lock_wait=None, local_path='borg', remote_path=None):
|
||||
'''
|
||||
Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
|
||||
the dry-run.
|
||||
'''
|
||||
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
||||
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
|
||||
verbosity_flags = {
|
||||
VERBOSITY_SOME: ('--info',),
|
||||
VERBOSITY_LOTS: ('--debug',),
|
||||
|
@ -23,7 +24,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot
|
|||
local_path, 'list',
|
||||
'--short',
|
||||
repository,
|
||||
) + remote_path_flags + verbosity_flags
|
||||
) + remote_path_flags + lock_wait_flags + verbosity_flags
|
||||
|
||||
list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
|
||||
|
||||
|
@ -39,7 +40,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot
|
|||
repository=repository,
|
||||
last_archive_name=last_archive_name,
|
||||
),
|
||||
) + remote_path_flags + verbosity_flags + list_flag
|
||||
) + remote_path_flags + lock_wait_flags + verbosity_flags + list_flag
|
||||
|
||||
logger.debug(' '.join(full_extract_command))
|
||||
subprocess.check_call(full_extract_command)
|
||||
|
|
|
@ -32,12 +32,16 @@ def _make_prune_flags(retention_config):
|
|||
)
|
||||
|
||||
|
||||
def prune_archives(verbosity, dry_run, repository, retention_config, local_path='borg', remote_path=None):
|
||||
def prune_archives(verbosity, dry_run, repository, storage_config, retention_config,
|
||||
local_path='borg', remote_path=None):
|
||||
'''
|
||||
Given verbosity/dry-run flags, a local or remote repository path, a retention config dict, prune
|
||||
Borg archives according the the retention policy specified in that configuration.
|
||||
Given verbosity/dry-run flags, a local or remote repository path, a storage config dict, and a
|
||||
retention config dict, prune Borg archives according the the retention policy specified in that
|
||||
configuration.
|
||||
'''
|
||||
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
||||
lock_wait = storage_config.get('lock_wait', None)
|
||||
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
|
||||
verbosity_flags = {
|
||||
VERBOSITY_SOME: ('--info', '--stats',),
|
||||
VERBOSITY_LOTS: ('--debug', '--stats', '--list'),
|
||||
|
@ -51,7 +55,7 @@ def prune_archives(verbosity, dry_run, repository, retention_config, local_path=
|
|||
element
|
||||
for pair in _make_prune_flags(retention_config)
|
||||
for element in pair
|
||||
) + remote_path_flags + verbosity_flags + dry_run_flags
|
||||
) + remote_path_flags + lock_wait_flags + verbosity_flags + dry_run_flags
|
||||
|
||||
logger.debug(' '.join(full_command))
|
||||
subprocess.check_call(full_command)
|
||||
|
|
|
@ -113,6 +113,7 @@ def run_configuration(config_filename, args): # pragma: no cover
|
|||
args.verbosity,
|
||||
args.dry_run,
|
||||
repository,
|
||||
storage,
|
||||
retention,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
|
@ -133,6 +134,7 @@ def run_configuration(config_filename, args): # pragma: no cover
|
|||
check.check_archives(
|
||||
args.verbosity,
|
||||
repository,
|
||||
storage,
|
||||
consistency,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
|
|
|
@ -139,6 +139,10 @@ map:
|
|||
type: scalar
|
||||
desc: Umask to be used for borg create.
|
||||
example: 0077
|
||||
lock_wait:
|
||||
type: int
|
||||
desc: Maximum seconds to wait for acquiring a repository/cache lock.
|
||||
example: 5
|
||||
archive_name_format:
|
||||
type: scalar
|
||||
desc: |
|
||||
|
@ -152,6 +156,7 @@ map:
|
|||
desc: |
|
||||
Retention policy for how many backups to keep in each category. See
|
||||
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
|
||||
At least one of the "keep" options is required for pruning to work.
|
||||
map:
|
||||
keep_within:
|
||||
type: scalar
|
||||
|
|
|
@ -103,6 +103,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
|
|||
module.check_archives(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
consistency_config=consistency_config,
|
||||
)
|
||||
|
||||
|
@ -119,6 +120,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
|
|||
module.check_archives(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
consistency_config=consistency_config,
|
||||
)
|
||||
|
||||
|
@ -136,6 +138,7 @@ def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
|
|||
module.check_archives(
|
||||
verbosity=VERBOSITY_SOME,
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
consistency_config=consistency_config,
|
||||
)
|
||||
|
||||
|
@ -153,6 +156,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
|
|||
module.check_archives(
|
||||
verbosity=VERBOSITY_LOTS,
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
consistency_config=consistency_config,
|
||||
)
|
||||
|
||||
|
@ -165,6 +169,7 @@ def test_check_archives_without_any_checks_bails():
|
|||
module.check_archives(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
consistency_config=consistency_config,
|
||||
)
|
||||
|
||||
|
@ -186,6 +191,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
|
|||
module.check_archives(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
consistency_config=consistency_config,
|
||||
local_path='borg1',
|
||||
)
|
||||
|
@ -208,6 +214,29 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
|
|||
module.check_archives(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
consistency_config=consistency_config,
|
||||
remote_path='borg1',
|
||||
)
|
||||
|
||||
|
||||
def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
|
||||
checks = ('repository',)
|
||||
check_last = flexmock()
|
||||
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
|
||||
stdout = flexmock()
|
||||
insert_subprocess_mock(
|
||||
('borg', 'check', 'repo', '--lock-wait', '5'),
|
||||
stdout=stdout, stderr=STDOUT,
|
||||
)
|
||||
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
|
||||
flexmock(module.os).should_receive('devnull')
|
||||
|
||||
module.check_archives(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
storage_config={'lock_wait': 5},
|
||||
consistency_config=consistency_config,
|
||||
)
|
||||
|
|
|
@ -518,6 +518,26 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
|
|||
)
|
||||
|
||||
|
||||
def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||
insert_subprocess_mock(CREATE_COMMAND + ('--lock-wait', '5'))
|
||||
|
||||
module.create_archive(
|
||||
verbosity=None,
|
||||
dry_run=False,
|
||||
repository='repo',
|
||||
location_config={
|
||||
'source_directories': ['foo', 'bar'],
|
||||
'repositories': ['repo'],
|
||||
'exclude_patterns': None,
|
||||
},
|
||||
storage_config={'lock_wait': 5},
|
||||
)
|
||||
|
||||
|
||||
def test_create_archive_with_source_directories_glob_expands():
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
|
|
@ -34,6 +34,7 @@ def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
|
|||
module.extract_last_archive_dry_run(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
lock_wait=None,
|
||||
)
|
||||
|
||||
|
||||
|
@ -48,6 +49,7 @@ def test_extract_last_archive_dry_run_without_any_archives_should_bail():
|
|||
module.extract_last_archive_dry_run(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
lock_wait=None,
|
||||
)
|
||||
|
||||
|
||||
|
@ -64,6 +66,7 @@ def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_
|
|||
module.extract_last_archive_dry_run(
|
||||
verbosity=VERBOSITY_SOME,
|
||||
repository='repo',
|
||||
lock_wait=None,
|
||||
)
|
||||
|
||||
|
||||
|
@ -80,6 +83,7 @@ def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_
|
|||
module.extract_last_archive_dry_run(
|
||||
verbosity=VERBOSITY_LOTS,
|
||||
repository='repo',
|
||||
lock_wait=None,
|
||||
)
|
||||
|
||||
|
||||
|
@ -96,6 +100,7 @@ def test_extract_last_archive_dry_run_should_call_borg_via_local_path():
|
|||
module.extract_last_archive_dry_run(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
lock_wait=None,
|
||||
local_path='borg1',
|
||||
)
|
||||
|
||||
|
@ -113,5 +118,23 @@ def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_paramete
|
|||
module.extract_last_archive_dry_run(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
lock_wait=None,
|
||||
remote_path='borg1',
|
||||
)
|
||||
|
||||
|
||||
def test_extract_last_archive_dry_run_should_call_borg_with_lock_wait_parameters():
|
||||
flexmock(sys.stdout).encoding = 'utf-8'
|
||||
insert_subprocess_check_output_mock(
|
||||
('borg', 'list', '--short', 'repo', '--lock-wait', '5'),
|
||||
result='archive1\narchive2\n'.encode('utf-8'),
|
||||
)
|
||||
insert_subprocess_mock(
|
||||
('borg', 'extract', '--dry-run', 'repo::archive2', '--lock-wait', '5'),
|
||||
)
|
||||
|
||||
module.extract_last_archive_dry_run(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
lock_wait=5,
|
||||
)
|
||||
|
|
|
@ -66,6 +66,7 @@ def test_prune_archives_calls_borg_with_parameters():
|
|||
verbosity=None,
|
||||
dry_run=False,
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
retention_config=retention_config,
|
||||
)
|
||||
|
||||
|
@ -79,6 +80,7 @@ def test_prune_archives_with_verbosity_some_calls_borg_with_info_parameter():
|
|||
|
||||
module.prune_archives(
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
verbosity=VERBOSITY_SOME,
|
||||
dry_run=False,
|
||||
retention_config=retention_config,
|
||||
|
@ -94,6 +96,7 @@ def test_prune_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
|
|||
|
||||
module.prune_archives(
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
verbosity=VERBOSITY_LOTS,
|
||||
dry_run=False,
|
||||
retention_config=retention_config,
|
||||
|
@ -109,6 +112,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter():
|
|||
|
||||
module.prune_archives(
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
verbosity=None,
|
||||
dry_run=True,
|
||||
retention_config=retention_config,
|
||||
|
@ -126,6 +130,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path():
|
|||
verbosity=None,
|
||||
dry_run=False,
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
retention_config=retention_config,
|
||||
local_path='borg1',
|
||||
)
|
||||
|
@ -142,6 +147,24 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters(
|
|||
verbosity=None,
|
||||
dry_run=False,
|
||||
repository='repo',
|
||||
storage_config={},
|
||||
retention_config=retention_config,
|
||||
remote_path='borg1',
|
||||
)
|
||||
|
||||
|
||||
def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
|
||||
storage_config = {'lock_wait': 5}
|
||||
retention_config = flexmock()
|
||||
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
|
||||
BASE_PRUNE_FLAGS,
|
||||
)
|
||||
insert_subprocess_mock(PRUNE_COMMAND + ('--lock-wait', '5'))
|
||||
|
||||
module.prune_archives(
|
||||
verbosity=None,
|
||||
dry_run=False,
|
||||
repository='repo',
|
||||
storage_config=storage_config,
|
||||
retention_config=retention_config,
|
||||
)
|
||||
|
|
2
setup.py
2
setup.py
|
@ -1,7 +1,7 @@
|
|||
from setuptools import setup, find_packages
|
||||
|
||||
|
||||
VERSION = '1.1.15.dev'
|
||||
VERSION = '1.1.15'
|
||||
|
||||
|
||||
setup(
|
||||
|
|
Loading…
Reference in a new issue