Add timeout and tests

This commit is contained in:
cadamswaite 2021-11-14 22:15:22 +00:00
parent 19cad89978
commit b4117916b8
3 changed files with 59 additions and 0 deletions

View file

@ -4,6 +4,7 @@ import json
import logging import logging
import os import os
import sys import sys
import time
from queue import Queue from queue import Queue
from subprocess import CalledProcessError from subprocess import CalledProcessError
@ -54,6 +55,7 @@ def run_configuration(config_filename, config, arguments):
local_path = location.get('local_path', 'borg') local_path = location.get('local_path', 'borg')
remote_path = location.get('remote_path') remote_path = location.get('remote_path')
retries = storage.get('retries', 0) retries = storage.get('retries', 0)
retry_timeout = storage.get('retry_timeout', 0)
borg_environment.initialize(storage) borg_environment.initialize(storage)
encountered_error = None encountered_error = None
error_repository = '' error_repository = ''
@ -130,6 +132,10 @@ def run_configuration(config_filename, config, arguments):
while not repo_queue.empty(): while not repo_queue.empty():
repository_path, retry_num = repo_queue.get() repository_path, retry_num = repo_queue.get()
timeout = retry_num * retry_timeout
if timeout:
logger.warning(f'Sleeping {timeout}s before next retry')
time.sleep(timeout)
try: try:
yield from run_actions( yield from run_actions(
arguments=arguments, arguments=arguments,

View file

@ -257,6 +257,12 @@ properties:
Number of times to retry a backup before failing. Defaults Number of times to retry a backup before failing. Defaults
to 0 (i.e. does not attempt retry). to 0 (i.e. does not attempt retry).
example: 3 example: 3
retry_timeout:
type: integer
description: |
Wait time between retries, to allow transient issues to pass.
Defaults to 0s.
example: 10
temporary_directory: temporary_directory:
type: string type: string
description: | description: |

View file

@ -1,5 +1,6 @@
import logging import logging
import subprocess import subprocess
import time
from flexmock import flexmock from flexmock import flexmock
@ -257,6 +258,52 @@ def test_run_retries_one_passes():
results = list(module.run_configuration('test.yaml', config, arguments)) results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results assert results == expected_results
def test_run_retry_timeout():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
flexmock(module).should_receive('make_error_log_records') \
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered()
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
flexmock(module).should_receive('make_error_log_records') \
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered()
flexmock(time).should_receive('sleep').with_args(20).and_return().ordered()
flexmock(module).should_receive('make_error_log_records') \
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered()
flexmock(time).should_receive('sleep').with_args(30).and_return().ordered()
flexmock(module).should_receive('make_error_log_records') \
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[3:4]).ordered()
config = {'location': {'repositories': ['foo']}, 'storage': {'retries':3, 'retry_timeout':10}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_run_retries_timeout_multiple_repos():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return([]).and_raise(OSError).times(4)
expected_results = [flexmock(), flexmock(), flexmock()]
flexmock(module).should_receive('make_error_log_records') \
.with_args('foo: Error running actions for repository', OSError).and_return(expected_results[0:1]).ordered()
flexmock(module).should_receive('make_error_log_records') \
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[1:2]).ordered()
# Sleep before retrying foo (and passing)
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
# Sleep before retrying bar (and failing)
flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
flexmock(module).should_receive('make_error_log_records') \
.with_args('bar: Error running actions for repository', OSError).and_return(expected_results[2:3]).ordered()
config = {'location': {'repositories': ['foo','bar']}, 'storage': {'retries':1, 'retry_timeout':10}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_load_configurations_collects_parsed_configurations(): def test_load_configurations_collects_parsed_configurations():
configuration = flexmock() configuration = flexmock()
other_configuration = flexmock() other_configuration = flexmock()