psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Formatting

Browse source
This commit is contained in:
cadamswaite 2021-11-14 22:37:42 +00:00
parent b4117916b8
commit 976a877a25
3 changed files with 83 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
borgmatic/commands/borgmatic.py
View file

@ -126,9 +126,7 @@ def run_configuration(config_filename, config, arguments):
if not encountered_error: if not encountered_error:
repo_queue = Queue() repo_queue = Queue()
for repo in location['repositories']: for repo in location['repositories']:
repo_queue.put( repo_queue.put((repo, 0),)
(repo, 0),
)
while not repo_queue.empty(): while not repo_queue.empty():
repository_path, retry_num = repo_queue.get() repository_path, retry_num = repo_queue.get()
@ -153,9 +151,7 @@ def run_configuration(config_filename, config, arguments):
'{}: Error running actions for repository'.format(repository_path), error '{}: Error running actions for repository'.format(repository_path), error
) )
if retry_num < retries: if retry_num < retries:
repo_queue.put( repo_queue.put((repository_path, retry_num + 1),)
(repository_path, retry_num + 1),
)
logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}') logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}')
continue continue
encountered_error = error encountered_error = error

2
borgmatic/config/schema.yaml
View file

@ -260,7 +260,7 @@ properties:
retry_timeout: retry_timeout:
type: integer type: integer
description: | description: |
Wait time between retries, to allow transient issues to pass. Wait time between retries, to allow transient issues to pass
Defaults to 0s. Defaults to 0s.
example: 10 example: 10
temporary_directory: temporary_directory:

111
tests/unit/commands/test_borgmatic.py
View file

@ -184,6 +184,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure():
assert results == expected_results assert results == expected_results
def test_run_retries_soft_error(): def test_run_retries_soft_error():
# Run action first fails, second passes # Run action first fails, second passes
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
@ -196,114 +197,148 @@ def test_run_retries_soft_error():
results = list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results assert results == expected_results
def test_run_retries_hard_error(): def test_run_retries_hard_error():
# Run action fails twice # Run action fails twice
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
expected_results = [flexmock(), flexmock()] expected_results = [flexmock(), flexmock()]
flexmock(module).should_receive('make_error_log_records') \ flexmock(module).should_receive('make_error_log_records').with_args(
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[:1]) \ 'foo: Error running actions for repository', OSError
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:]).twice() ).and_return(expected_results[:1]).with_args(
'foo: Error running actions for repository', OSError
).and_return(
expected_results[1:]
).twice()
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}} config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results assert results == expected_results
def test_run_repos_ordered(): def test_run_repos_ordered():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
expected_results = [flexmock(), flexmock()] expected_results = [flexmock(), flexmock()]
flexmock(module).should_receive('make_error_log_records') \ flexmock(module).should_receive('make_error_log_records').with_args(
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[:1]).ordered() 'foo: Error running actions for repository', OSError
flexmock(module).should_receive('make_error_log_records') \ ).and_return(expected_results[:1]).ordered()
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:]).ordered() flexmock(module).should_receive('make_error_log_records').with_args(
'bar: Error running actions for repository', OSError
).and_return(expected_results[1:]).ordered()
config = {'location': {'repositories': ['foo', 'bar']}} config = {'location': {'repositories': ['foo', 'bar']}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results assert results == expected_results
def test_run_retries_round_robbin(): def test_run_retries_round_robbin():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
expected_results = [flexmock(), flexmock(), flexmock(), flexmock()] expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
flexmock(module).should_receive('make_error_log_records') \ flexmock(module).should_receive('make_error_log_records').with_args(
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() 'foo: Error running actions for repository', OSError
flexmock(module).should_receive('make_error_log_records') \ ).and_return(expected_results[0:1]).ordered()
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() flexmock(module).should_receive('make_error_log_records').with_args(
flexmock(module).should_receive('make_error_log_records') \ 'bar: Error running actions for repository', OSError
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() ).and_return(expected_results[1:2]).ordered()
flexmock(module).should_receive('make_error_log_records') \ flexmock(module).should_receive('make_error_log_records').with_args(
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered() 'foo: Error running actions for repository', OSError
).and_return(expected_results[2:3]).ordered()
flexmock(module).should_receive('make_error_log_records').with_args(
'bar: Error running actions for repository', OSError
).and_return(expected_results[3:4]).ordered()
config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results assert results == expected_results
def test_run_retries_one_passes(): def test_run_retries_one_passes():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4) flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
[]
).and_raise(OSError).times(4)
expected_results = [flexmock(), flexmock(), flexmock()] expected_results = [flexmock(), flexmock(), flexmock()]
flexmock(module).should_receive('make_error_log_records') \ flexmock(module).should_receive('make_error_log_records').with_args(
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() 'foo: Error running actions for repository', OSError
flexmock(module).should_receive('make_error_log_records') \ ).and_return(expected_results[0:1]).ordered()
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() flexmock(module).should_receive('make_error_log_records').with_args(
flexmock(module).should_receive('make_error_log_records') \ 'bar: Error running actions for repository', OSError
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() ).and_return(expected_results[1:2]).ordered()
flexmock(module).should_receive('make_error_log_records').with_args(
'bar: Error running actions for repository', OSError
).and_return(expected_results[2:3]).ordered()
config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}} config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results assert results == expected_results
def test_run_retry_timeout(): def test_run_retry_timeout():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
expected_results = [flexmock(), flexmock(), flexmock(), flexmock()] expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
flexmock(module).should_receive('make_error_log_records') \ flexmock(module).should_receive('make_error_log_records').with_args(
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() 'foo: Error running actions for repository', OSError
).and_return(expected_results[0:1]).ordered()
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
flexmock(module).should_receive('make_error_log_records') \ flexmock(module).should_receive('make_error_log_records').with_args(
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() 'foo: Error running actions for repository', OSError
).and_return(expected_results[1:2]).ordered()
flexmock(time).should_receive('sleep').with_args(20).and_return().ordered() flexmock(time).should_receive('sleep').with_args(20).and_return().ordered()
flexmock(module).should_receive('make_error_log_records') \ flexmock(module).should_receive('make_error_log_records').with_args(
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() 'foo: Error running actions for repository', OSError
).and_return(expected_results[2:3]).ordered()
flexmock(time).should_receive('sleep').with_args(30).and_return().ordered() flexmock(time).should_receive('sleep').with_args(30).and_return().ordered()
flexmock(module).should_receive('make_error_log_records') \ flexmock(module).should_receive('make_error_log_records').with_args(
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered() 'foo: Error running actions for repository', OSError
).and_return(expected_results[3:4]).ordered()
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_timeout': 10}} config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_timeout': 10}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results assert results == expected_results
def test_run_retries_timeout_multiple_repos(): def test_run_retries_timeout_multiple_repos():
flexmock(module.borg_environment).should_receive('initialize') flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook') flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4) flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
[]
).and_raise(OSError).times(4)
expected_results = [flexmock(), flexmock(), flexmock()] expected_results = [flexmock(), flexmock(), flexmock()]
flexmock(module).should_receive('make_error_log_records') \ flexmock(module).should_receive('make_error_log_records').with_args(
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered() 'foo: Error running actions for repository', OSError
flexmock(module).should_receive('make_error_log_records') \ ).and_return(expected_results[0:1]).ordered()
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered() flexmock(module).should_receive('make_error_log_records').with_args(
'bar: Error running actions for repository', OSError
).and_return(expected_results[1:2]).ordered()
# Sleep before retrying foo (and passing) # Sleep before retrying foo (and passing)
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
# Sleep before retrying bar (and failing) # Sleep before retrying bar (and failing)
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
flexmock(module).should_receive('make_error_log_records') \ flexmock(module).should_receive('make_error_log_records').with_args(
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered() 'bar: Error running actions for repository', OSError
config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1, 'retry_timeout':10}} ).and_return(expected_results[2:3]).ordered()
config = {
'location': {'repositories': ['foo', 'bar']},
'storage': {'retries': 1, 'retry_timeout': 10},
}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results assert results == expected_results
def test_load_configurations_collects_parsed_configurations(): def test_load_configurations_collects_parsed_configurations():
configuration = flexmock() configuration = flexmock()
other_configuration = flexmock() other_configuration = flexmock()