Add timeout and tests
This commit is contained in:
parent
19cad89978
commit
b4117916b8
3 changed files with 59 additions and 0 deletions
|
@ -4,6 +4,7 @@ import json
|
|||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from queue import Queue
|
||||
from subprocess import CalledProcessError
|
||||
|
||||
|
@ -54,6 +55,7 @@ def run_configuration(config_filename, config, arguments):
|
|||
local_path = location.get('local_path', 'borg')
|
||||
remote_path = location.get('remote_path')
|
||||
retries = storage.get('retries', 0)
|
||||
retry_timeout = storage.get('retry_timeout', 0)
|
||||
borg_environment.initialize(storage)
|
||||
encountered_error = None
|
||||
error_repository = ''
|
||||
|
@ -130,6 +132,10 @@ def run_configuration(config_filename, config, arguments):
|
|||
|
||||
while not repo_queue.empty():
|
||||
repository_path, retry_num = repo_queue.get()
|
||||
timeout = retry_num * retry_timeout
|
||||
if timeout:
|
||||
logger.warning(f'Sleeping {timeout}s before next retry')
|
||||
time.sleep(timeout)
|
||||
try:
|
||||
yield from run_actions(
|
||||
arguments=arguments,
|
||||
|
|
|
@ -257,6 +257,12 @@ properties:
|
|||
Number of times to retry a backup before failing. Defaults
|
||||
to 0 (i.e. does not attempt retry).
|
||||
example: 3
|
||||
retry_timeout:
|
||||
type: integer
|
||||
description: |
|
||||
Wait time between retries, to allow transient issues to pass.
|
||||
Defaults to 0s.
|
||||
example: 10
|
||||
temporary_directory:
|
||||
type: string
|
||||
description: |
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
import logging
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from flexmock import flexmock
|
||||
|
||||
|
@ -257,6 +258,52 @@ def test_run_retries_one_passes():
|
|||
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||
assert results == expected_results
|
||||
|
||||
def test_run_retry_timeout():
|
||||
flexmock(module.borg_environment).should_receive('initialize')
|
||||
flexmock(module.command).should_receive('execute_hook')
|
||||
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
|
||||
expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
|
||||
flexmock(module).should_receive('make_error_log_records') \
|
||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered()
|
||||
|
||||
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
|
||||
flexmock(module).should_receive('make_error_log_records') \
|
||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered()
|
||||
|
||||
flexmock(time).should_receive('sleep').with_args(20).and_return().ordered()
|
||||
flexmock(module).should_receive('make_error_log_records') \
|
||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered()
|
||||
|
||||
flexmock(time).should_receive('sleep').with_args(30).and_return().ordered()
|
||||
flexmock(module).should_receive('make_error_log_records') \
|
||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered()
|
||||
config = {'location': {'repositories': ['foo']}, 'storage': {'retries':3, 'retry_timeout':10}}
|
||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||
assert results == expected_results
|
||||
|
||||
def test_run_retries_timeout_multiple_repos():
|
||||
flexmock(module.borg_environment).should_receive('initialize')
|
||||
flexmock(module.command).should_receive('execute_hook')
|
||||
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4)
|
||||
expected_results = [flexmock(), flexmock(), flexmock()]
|
||||
flexmock(module).should_receive('make_error_log_records') \
|
||||
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered()
|
||||
flexmock(module).should_receive('make_error_log_records') \
|
||||
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered()
|
||||
|
||||
# Sleep before retrying foo (and passing)
|
||||
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
|
||||
|
||||
# Sleep before retrying bar (and failing)
|
||||
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
|
||||
flexmock(module).should_receive('make_error_log_records') \
|
||||
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered()
|
||||
config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1, 'retry_timeout':10}}
|
||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||
assert results == expected_results
|
||||
|
||||
def test_load_configurations_collects_parsed_configurations():
|
||||
configuration = flexmock()
|
||||
other_configuration = flexmock()
|
||||
|
|
Loading…
Reference in a new issue