psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support for Borg --lock-wait option for the maximum wait for a repository/cache lock (#56).

Browse source
This commit is contained in:
Dan Helfman 2018-02-19 15:51:04 -08:00
parent a87036ee46
commit 2d3f5fa05d
12 changed files with 132 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
NEWS
View file

@ -1,7 +1,9 @@
1.1.15.dev0
1.1.15
* Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file.
* Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together.
* #55: Fix for missing tags/releases from Gitea and GitHub project hosting.
Work-around for behavior introduced in Borg 1.1.3: https://github.com/borgbackup/borg/issues/3298
* #55: Fix for missing tags/releases on Gitea and GitHub project hosting.
* #56: Support for Borg --lock-wait option for the maximum wait for a repository/cache lock.
* #58: Support for using tilde in exclude_patterns to reference home directory.
1.1.14

15
borgmatic/borg/check.py
View file

@ -60,18 +60,23 @@ def _make_check_flags(checks, check_last=None):
) + last_flag
def check_archives(verbosity, repository, consistency_config, local_path='borg', remote_path=None):
def check_archives(verbosity, repository, storage_config, consistency_config, local_path='borg',
remote_path=None):
'''
Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
local/remote commands to run, check the contained Borg archives for consistency.
Given a verbosity flag, a local or remote repository path, a storage config dict, a consistency
config dict, and a local/remote commands to run, check the contained Borg archives for
consistency.
If there are no consistency checks to run, skip running them.
'''
checks = _parse_checks(consistency_config)
check_last = consistency_config.get('check_last', None)
lock_wait = None
if set(checks).intersection(set(DEFAULT_CHECKS)):
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
lock_wait = storage_config.get('lock_wait', None)
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
verbosity_flags = {
VERBOSITY_SOME: ('--info',),
VERBOSITY_LOTS: ('--debug',),
@ -80,7 +85,7 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg',
full_command = (
local_path, 'check',
repository,
) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
) + _make_check_flags(checks, check_last) + remote_path_flags + lock_wait_flags + verbosity_flags
# The check command spews to stdout/stderr even without the verbose flag. Suppress it.
stdout = None if verbosity_flags else open(os.devnull, 'w')
@ -89,4 +94,4 @@ def check_archives(verbosity, repository, consistency_config, local_path='borg',
subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
if 'extract' in checks:
extract.extract_last_archive_dry_run(verbosity, repository, local_path, remote_path)
extract.extract_last_archive_dry_run(verbosity, repository, lock_wait, local_path, remote_path)

4
borgmatic/borg/create.py
View file

@ -129,6 +129,8 @@ def create_archive(
remote_rate_limit_flags = ('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ()
umask = storage_config.get('umask', None)
umask_flags = ('--umask', str(umask)) if umask else ()
lock_wait = storage_config.get('lock_wait', None)
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else ()
files_cache = location_config.get('files_cache')
files_cache_flags = ('--files-cache', files_cache) if files_cache else ()
@ -149,7 +151,7 @@ def create_archive(
),
) + sources + pattern_flags + exclude_flags + compression_flags + remote_rate_limit_flags + \
one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \
verbosity_flags + dry_run_flags
lock_wait_flags + verbosity_flags + dry_run_flags
logger.debug(' '.join(full_command))
subprocess.check_call(full_command)

7
borgmatic/borg/extract.py
View file

@ -8,12 +8,13 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
logger = logging.getLogger(__name__)
def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remote_path=None):
def extract_last_archive_dry_run(verbosity, repository, lock_wait=None, local_path='borg', remote_path=None):
'''
Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
the dry-run.
'''
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
verbosity_flags = {
VERBOSITY_SOME: ('--info',),
VERBOSITY_LOTS: ('--debug',),
@ -23,7 +24,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot
local_path, 'list',
'--short',
repository,
) + remote_path_flags + verbosity_flags
) + remote_path_flags + lock_wait_flags + verbosity_flags
list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
@ -39,7 +40,7 @@ def extract_last_archive_dry_run(verbosity, repository, local_path='borg', remot
repository=repository,
last_archive_name=last_archive_name,
),
) + remote_path_flags + verbosity_flags + list_flag
) + remote_path_flags + lock_wait_flags + verbosity_flags + list_flag
logger.debug(' '.join(full_extract_command))
subprocess.check_call(full_extract_command)

12
borgmatic/borg/prune.py
View file

@ -32,12 +32,16 @@ def _make_prune_flags(retention_config):
)
def prune_archives(verbosity, dry_run, repository, retention_config, local_path='borg', remote_path=None):
def prune_archives(verbosity, dry_run, repository, storage_config, retention_config,
local_path='borg', remote_path=None):
'''
Given verbosity/dry-run flags, a local or remote repository path, a retention config dict, prune
Borg archives according the the retention policy specified in that configuration.
Given verbosity/dry-run flags, a local or remote repository path, a storage config dict, and a
retention config dict, prune Borg archives according the the retention policy specified in that
configuration.
'''
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
lock_wait = storage_config.get('lock_wait', None)
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
verbosity_flags = {
VERBOSITY_SOME: ('--info', '--stats',),
VERBOSITY_LOTS: ('--debug', '--stats', '--list'),
@ -51,7 +55,7 @@ def prune_archives(verbosity, dry_run, repository, retention_config, local_path=
element
for pair in _make_prune_flags(retention_config)
for element in pair
) + remote_path_flags + verbosity_flags + dry_run_flags
) + remote_path_flags + lock_wait_flags + verbosity_flags + dry_run_flags
logger.debug(' '.join(full_command))
subprocess.check_call(full_command)

2
borgmatic/commands/borgmatic.py
View file

@ -113,6 +113,7 @@ def run_configuration(config_filename, args): # pragma: no cover
args.verbosity,
args.dry_run,
repository,
storage,
retention,
local_path=local_path,
remote_path=remote_path,
@ -133,6 +134,7 @@ def run_configuration(config_filename, args): # pragma: no cover
check.check_archives(
args.verbosity,
repository,
storage,
consistency,
local_path=local_path,
remote_path=remote_path,

5
borgmatic/config/schema.yaml
View file

@ -139,6 +139,10 @@ map:
type: scalar
desc: Umask to be used for borg create.
example: 0077
lock_wait:
type: int
desc: Maximum seconds to wait for acquiring a repository/cache lock.
example: 5
archive_name_format:
type: scalar
desc: |
@ -152,6 +156,7 @@ map:
desc: |
Retention policy for how many backups to keep in each category. See
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
At least one of the "keep" options is required for pruning to work.
map:
keep_within:
type: scalar

29
borgmatic/tests/unit/borg/test_check.py
View file

@ -103,6 +103,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
module.check_archives(
verbosity=None,
repository='repo',
storage_config={},
consistency_config=consistency_config,
)
@ -119,6 +120,7 @@ def test_check_archives_with_extract_check_calls_extract_only():
module.check_archives(
verbosity=None,
repository='repo',
storage_config={},
consistency_config=consistency_config,
)
@ -136,6 +138,7 @@ def test_check_archives_with_verbosity_some_calls_borg_with_info_parameter():
module.check_archives(
verbosity=VERBOSITY_SOME,
repository='repo',
storage_config={},
consistency_config=consistency_config,
)
@ -153,6 +156,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
module.check_archives(
verbosity=VERBOSITY_LOTS,
repository='repo',
storage_config={},
consistency_config=consistency_config,
)
@ -165,6 +169,7 @@ def test_check_archives_without_any_checks_bails():
module.check_archives(
verbosity=None,
repository='repo',
storage_config={},
consistency_config=consistency_config,
)
@ -186,6 +191,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
module.check_archives(
verbosity=None,
repository='repo',
storage_config={},
consistency_config=consistency_config,
local_path='borg1',
)
@ -208,6 +214,29 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
module.check_archives(
verbosity=None,
repository='repo',
storage_config={},
consistency_config=consistency_config,
remote_path='borg1',
)
def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
checks = ('repository',)
check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
stdout = flexmock()
insert_subprocess_mock(
('borg', 'check', 'repo', '--lock-wait', '5'),
stdout=stdout, stderr=STDOUT,
)
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull')
module.check_archives(
verbosity=None,
repository='repo',
storage_config={'lock_wait': 5},
consistency_config=consistency_config,
)

20
borgmatic/tests/unit/borg/test_create.py
View file

@ -518,6 +518,26 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
)
def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + ('--lock-wait', '5'))
module.create_archive(
verbosity=None,
dry_run=False,
repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'exclude_patterns': None,
},
storage_config={'lock_wait': 5},
)
def test_create_archive_with_source_directories_glob_expands():
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)

23
borgmatic/tests/unit/borg/test_extract.py
View file

@ -34,6 +34,7 @@ def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
module.extract_last_archive_dry_run(
verbosity=None,
repository='repo',
lock_wait=None,
)
@ -48,6 +49,7 @@ def test_extract_last_archive_dry_run_without_any_archives_should_bail():
module.extract_last_archive_dry_run(
verbosity=None,
repository='repo',
lock_wait=None,
)
@ -64,6 +66,7 @@ def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_
module.extract_last_archive_dry_run(
verbosity=VERBOSITY_SOME,
repository='repo',
lock_wait=None,
)
@ -80,6 +83,7 @@ def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_
module.extract_last_archive_dry_run(
verbosity=VERBOSITY_LOTS,
repository='repo',
lock_wait=None,
)
@ -96,6 +100,7 @@ def test_extract_last_archive_dry_run_should_call_borg_via_local_path():
module.extract_last_archive_dry_run(
verbosity=None,
repository='repo',
lock_wait=None,
local_path='borg1',
)
@ -113,5 +118,23 @@ def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_paramete
module.extract_last_archive_dry_run(
verbosity=None,
repository='repo',
lock_wait=None,
remote_path='borg1',
)
def test_extract_last_archive_dry_run_should_call_borg_with_lock_wait_parameters():
flexmock(sys.stdout).encoding = 'utf-8'
insert_subprocess_check_output_mock(
('borg', 'list', '--short', 'repo', '--lock-wait', '5'),
result='archive1\narchive2\n'.encode('utf-8'),
)
insert_subprocess_mock(
('borg', 'extract', '--dry-run', 'repo::archive2', '--lock-wait', '5'),
)
module.extract_last_archive_dry_run(
verbosity=None,
repository='repo',
lock_wait=5,
)

23
borgmatic/tests/unit/borg/test_prune.py
View file

@ -66,6 +66,7 @@ def test_prune_archives_calls_borg_with_parameters():
verbosity=None,
dry_run=False,
repository='repo',
storage_config={},
retention_config=retention_config,
)
@ -79,6 +80,7 @@ def test_prune_archives_with_verbosity_some_calls_borg_with_info_parameter():
module.prune_archives(
repository='repo',
storage_config={},
verbosity=VERBOSITY_SOME,
dry_run=False,
retention_config=retention_config,
@ -94,6 +96,7 @@ def test_prune_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
module.prune_archives(
repository='repo',
storage_config={},
verbosity=VERBOSITY_LOTS,
dry_run=False,
retention_config=retention_config,
@ -109,6 +112,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter():
module.prune_archives(
repository='repo',
storage_config={},
verbosity=None,
dry_run=True,
retention_config=retention_config,
@ -126,6 +130,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path():
verbosity=None,
dry_run=False,
repository='repo',
storage_config={},
retention_config=retention_config,
local_path='borg1',
)
@ -142,6 +147,24 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters(
verbosity=None,
dry_run=False,
repository='repo',
storage_config={},
retention_config=retention_config,
remote_path='borg1',
)
def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
storage_config = {'lock_wait': 5}
retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS,
)
insert_subprocess_mock(PRUNE_COMMAND + ('--lock-wait', '5'))
module.prune_archives(
verbosity=None,
dry_run=False,
repository='repo',
storage_config=storage_config,
retention_config=retention_config,
)

2
setup.py
View file

@ -1,7 +1,7 @@
from setuptools import setup, find_packages
VERSION = '1.1.15.dev'
VERSION = '1.1.15'
setup(