Rename retry_timeout to retry_wait and standardize log formatting (#28).
This commit is contained in:
parent
180018fd81
commit
38ebfd2969
4 changed files with 21 additions and 17 deletions
1
NEWS
1
NEWS
|
@ -1,4 +1,5 @@
|
||||||
1.5.21.dev0
|
1.5.21.dev0
|
||||||
|
* #28: Optionally retry failing backups via "retries" and "retry_wait" configuration options.
|
||||||
* #459: Add support for old version (2.x) of jsonschema library.
|
* #459: Add support for old version (2.x) of jsonschema library.
|
||||||
|
|
||||||
1.5.20
|
1.5.20
|
||||||
|
|
|
@ -55,7 +55,7 @@ def run_configuration(config_filename, config, arguments):
|
||||||
local_path = location.get('local_path', 'borg')
|
local_path = location.get('local_path', 'borg')
|
||||||
remote_path = location.get('remote_path')
|
remote_path = location.get('remote_path')
|
||||||
retries = storage.get('retries', 0)
|
retries = storage.get('retries', 0)
|
||||||
retry_timeout = storage.get('retry_timeout', 0)
|
retry_wait = storage.get('retry_wait', 0)
|
||||||
borg_environment.initialize(storage)
|
borg_environment.initialize(storage)
|
||||||
encountered_error = None
|
encountered_error = None
|
||||||
error_repository = ''
|
error_repository = ''
|
||||||
|
@ -130,9 +130,9 @@ def run_configuration(config_filename, config, arguments):
|
||||||
|
|
||||||
while not repo_queue.empty():
|
while not repo_queue.empty():
|
||||||
repository_path, retry_num = repo_queue.get()
|
repository_path, retry_num = repo_queue.get()
|
||||||
timeout = retry_num * retry_timeout
|
timeout = retry_num * retry_wait
|
||||||
if timeout:
|
if timeout:
|
||||||
logger.warning(f'Sleeping {timeout}s before next retry')
|
logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry')
|
||||||
time.sleep(timeout)
|
time.sleep(timeout)
|
||||||
try:
|
try:
|
||||||
yield from run_actions(
|
yield from run_actions(
|
||||||
|
@ -152,7 +152,9 @@ def run_configuration(config_filename, config, arguments):
|
||||||
)
|
)
|
||||||
if retry_num < retries:
|
if retry_num < retries:
|
||||||
repo_queue.put((repository_path, retry_num + 1),)
|
repo_queue.put((repository_path, retry_num + 1),)
|
||||||
logger.warning(f'Retrying.. attempt {retry_num + 1}/{retries}')
|
logger.warning(
|
||||||
|
f'{config_filename}: Retrying... attempt {retry_num + 1}/{retries}'
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
encountered_error = error
|
encountered_error = error
|
||||||
error_repository = repository_path
|
error_repository = repository_path
|
||||||
|
|
|
@ -254,14 +254,15 @@ properties:
|
||||||
retries:
|
retries:
|
||||||
type: integer
|
type: integer
|
||||||
description: |
|
description: |
|
||||||
Number of times to retry a backup before failing. Defaults
|
Number of times to retry a failing backup before giving up.
|
||||||
to 0 (i.e. does not attempt retry).
|
Defaults to 0 (i.e., does not attempt retry).
|
||||||
example: 3
|
example: 3
|
||||||
retry_timeout:
|
retry_wait:
|
||||||
type: integer
|
type: integer
|
||||||
description: |
|
description: |
|
||||||
Wait time between retries, to allow transient issues to pass
|
Wait time between retries (in seconds) to allow transient
|
||||||
Defaults to 0s.
|
issues to pass. Increases after each retry as a form of
|
||||||
|
backoff. Defaults to 0 (no wait).
|
||||||
example: 10
|
example: 10
|
||||||
temporary_directory:
|
temporary_directory:
|
||||||
type: string
|
type: string
|
||||||
|
|
|
@ -185,7 +185,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure():
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retries_soft_error():
|
def test_run_configuration_retries_soft_error():
|
||||||
# Run action first fails, second passes
|
# Run action first fails, second passes
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
|
@ -198,7 +198,7 @@ def test_run_retries_soft_error():
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retries_hard_error():
|
def test_run_configuration_retries_hard_error():
|
||||||
# Run action fails twice
|
# Run action fails twice
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
|
@ -234,7 +234,7 @@ def test_run_repos_ordered():
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retries_round_robbin():
|
def test_run_configuration_retries_round_robbin():
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
|
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
|
||||||
|
@ -257,7 +257,7 @@ def test_run_retries_round_robbin():
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retries_one_passes():
|
def test_run_configuration_retries_one_passes():
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
|
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
|
||||||
|
@ -279,7 +279,7 @@ def test_run_retries_one_passes():
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retry_timeout():
|
def test_run_configuration_retry_wait():
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
|
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
|
||||||
|
@ -302,13 +302,13 @@ def test_run_retry_timeout():
|
||||||
flexmock(module).should_receive('make_error_log_records').with_args(
|
flexmock(module).should_receive('make_error_log_records').with_args(
|
||||||
'foo: Error running actions for repository', OSError
|
'foo: Error running actions for repository', OSError
|
||||||
).and_return(expected_results[3:4]).ordered()
|
).and_return(expected_results[3:4]).ordered()
|
||||||
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_timeout': 10}}
|
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_wait': 10}}
|
||||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_run_retries_timeout_multiple_repos():
|
def test_run_configuration_retries_timeout_multiple_repos():
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook')
|
flexmock(module.command).should_receive('execute_hook')
|
||||||
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
|
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
|
||||||
|
@ -332,7 +332,7 @@ def test_run_retries_timeout_multiple_repos():
|
||||||
).and_return(expected_results[2:3]).ordered()
|
).and_return(expected_results[2:3]).ordered()
|
||||||
config = {
|
config = {
|
||||||
'location': {'repositories': ['foo', 'bar']},
|
'location': {'repositories': ['foo', 'bar']},
|
||||||
'storage': {'retries': 1, 'retry_timeout': 10},
|
'storage': {'retries': 1, 'retry_wait': 10},
|
||||||
}
|
}
|
||||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
|
|
Loading…
Reference in a new issue