When backups to one of several repositories fails, keep backing up to the other repositories (#144).

This commit is contained in:
Dan Helfman 2019-09-30 22:19:31 -07:00
parent e14ebee4e0
commit 6c617eddd5
4 changed files with 225 additions and 104 deletions

4
NEWS
View file

@ -1,3 +1,7 @@
1.3.22
* #144: When backups to one of several repositories fails, keep backing up to the other
repositories and report errors afterwards.
1.3.21 1.3.21
* #192: User-defined hooks for global setup or cleanup that run before/after all actions. See the * #192: User-defined hooks for global setup or cleanup that run before/after all actions. See the
documentation for more information: documentation for more information:

View file

@ -28,13 +28,16 @@ logger = logging.getLogger(__name__)
LEGACY_CONFIG_PATH = '/etc/borgmatic/config' LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
def run_configuration(config_filename, config, arguments): # pragma: no cover def run_configuration(config_filename, config, arguments):
''' '''
Given a config filename, the corresponding parsed config dict, and command-line arguments as a Given a config filename, the corresponding parsed config dict, and command-line arguments as a
dict from subparser name to a namespace of parsed arguments, execute its defined pruning, dict from subparser name to a namespace of parsed arguments, execute its defined pruning,
backups, consistency checks, and/or other actions. backups, consistency checks, and/or other actions.
Yield JSON output strings from executing any actions that produce JSON. Yield a combination of:
* JSON output strings from successfully executing any actions that produce JSON
* logging.LogRecord instances containing errors from any actions or backup hooks that fail
''' '''
(location, storage, retention, consistency, hooks) = ( (location, storage, retention, consistency, hooks) = (
config.get(section_name, {}) config.get(section_name, {})
@ -42,12 +45,13 @@ def run_configuration(config_filename, config, arguments): # pragma: no cover
) )
global_arguments = arguments['global'] global_arguments = arguments['global']
try: local_path = location.get('local_path', 'borg')
local_path = location.get('local_path', 'borg') remote_path = location.get('remote_path')
remote_path = location.get('remote_path') borg_environment.initialize(storage)
borg_environment.initialize(storage) encountered_error = False
if 'create' in arguments: if 'create' in arguments:
try:
hook.execute_hook( hook.execute_hook(
hooks.get('before_backup'), hooks.get('before_backup'),
hooks.get('umask'), hooks.get('umask'),
@ -55,20 +59,33 @@ def run_configuration(config_filename, config, arguments): # pragma: no cover
'pre-backup', 'pre-backup',
global_arguments.dry_run, global_arguments.dry_run,
) )
except (OSError, CalledProcessError) as error:
for repository_path in location['repositories']: encountered_error = True
yield from run_actions( yield from make_error_log_records(
arguments=arguments, '{}: Error running pre-backup hook'.format(config_filename), error
location=location,
storage=storage,
retention=retention,
consistency=consistency,
local_path=local_path,
remote_path=remote_path,
repository_path=repository_path,
) )
if 'create' in arguments: if not encountered_error:
for repository_path in location['repositories']:
try:
yield from run_actions(
arguments=arguments,
location=location,
storage=storage,
retention=retention,
consistency=consistency,
local_path=local_path,
remote_path=remote_path,
repository_path=repository_path,
)
except (OSError, CalledProcessError) as error:
encountered_error = True
yield from make_error_log_records(
'{}: Error running actions for repository'.format(repository_path), error
)
if 'create' in arguments and not encountered_error:
try:
hook.execute_hook( hook.execute_hook(
hooks.get('after_backup'), hooks.get('after_backup'),
hooks.get('umask'), hooks.get('umask'),
@ -76,15 +93,25 @@ def run_configuration(config_filename, config, arguments): # pragma: no cover
'post-backup', 'post-backup',
global_arguments.dry_run, global_arguments.dry_run,
) )
except (OSError, CalledProcessError): except (OSError, CalledProcessError) as error:
hook.execute_hook( encountered_error = True
hooks.get('on_error'), yield from make_error_log_records(
hooks.get('umask'), '{}: Error running post-backup hook'.format(config_filename), error
config_filename, )
'on-error',
global_arguments.dry_run, if encountered_error:
) try:
raise hook.execute_hook(
hooks.get('on_error'),
hooks.get('umask'),
config_filename,
'on-error',
global_arguments.dry_run,
)
except (OSError, CalledProcessError) as error:
yield from make_error_log_records(
'{}: Error running on-error hook'.format(config_filename), error
)
def run_actions( def run_actions(
@ -231,11 +258,17 @@ def load_configurations(config_filenames):
return (configs, logs) return (configs, logs)
def make_error_log_records(error, message): def make_error_log_records(message, error=None):
''' '''
Given an exception object and error message text, yield a series of logging.LogRecord instances Given error message text and an optional exception object, yield a series of logging.LogRecord
with error summary information. instances with error summary information.
''' '''
if not error:
yield logging.makeLogRecord(
dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
)
return
try: try:
raise error raise error
except CalledProcessError as error: except CalledProcessError as error:
@ -279,25 +312,17 @@ def collect_configuration_run_summary_logs(configs, arguments):
try: try:
validate.guard_configuration_contains_repository(repository, configs) validate.guard_configuration_contains_repository(repository, configs)
except ValueError as error: except ValueError as error:
yield logging.makeLogRecord( yield from make_error_log_records(str(error))
dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
)
return return
if not configs: if not configs:
yield logging.makeLogRecord( yield from make_error_log_records(
dict( '{}: No configuration files found'.format(' '.join(arguments['global'].config_paths))
levelno=logging.CRITICAL,
levelname='CRITICAL',
msg='{}: No configuration files found'.format(
' '.join(arguments['global'].config_paths)
),
)
) )
return return
try: if 'create' in arguments:
if 'create' in arguments: try:
for config_filename, config in configs.items(): for config_filename, config in configs.items():
hooks = config.get('hooks', {}) hooks = config.get('hooks', {})
hook.execute_hook( hook.execute_hook(
@ -307,15 +332,22 @@ def collect_configuration_run_summary_logs(configs, arguments):
'pre-everything', 'pre-everything',
arguments['global'].dry_run, arguments['global'].dry_run,
) )
except (CalledProcessError, ValueError, OSError) as error: except (CalledProcessError, ValueError, OSError) as error:
yield from make_error_log_records(error, 'Error running pre-everything hook') yield from make_error_log_records('Error running pre-everything hook', error)
return return
# Execute the actions corresponding to each configuration file. # Execute the actions corresponding to each configuration file.
json_results = [] json_results = []
for config_filename, config in configs.items(): for config_filename, config in configs.items():
try: results = list(run_configuration(config_filename, config, arguments))
json_results.extend(list(run_configuration(config_filename, config, arguments))) error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord))
if error_logs:
yield from make_error_log_records(
'{}: Error running configuration file'.format(config_filename)
)
yield from error_logs
else:
yield logging.makeLogRecord( yield logging.makeLogRecord(
dict( dict(
levelno=logging.INFO, levelno=logging.INFO,
@ -323,16 +355,14 @@ def collect_configuration_run_summary_logs(configs, arguments):
msg='{}: Successfully ran configuration file'.format(config_filename), msg='{}: Successfully ran configuration file'.format(config_filename),
) )
) )
except (CalledProcessError, ValueError, OSError) as error: if results:
yield from make_error_log_records( json_results.extend(results)
error, '{}: Error running configuration file'.format(config_filename)
)
if json_results: if json_results:
sys.stdout.write(json.dumps(json_results)) sys.stdout.write(json.dumps(json_results))
try: if 'create' in arguments:
if 'create' in arguments: try:
for config_filename, config in configs.items(): for config_filename, config in configs.items():
hooks = config.get('hooks', {}) hooks = config.get('hooks', {})
hook.execute_hook( hook.execute_hook(
@ -342,8 +372,8 @@ def collect_configuration_run_summary_logs(configs, arguments):
'post-everything', 'post-everything',
arguments['global'].dry_run, arguments['global'].dry_run,
) )
except (CalledProcessError, ValueError, OSError) as error: except (CalledProcessError, ValueError, OSError) as error:
yield from make_error_log_records(error, 'Error running post-everything hook') yield from make_error_log_records('Error running post-everything hook', error)
def exit_with_help_link(): # pragma: no cover def exit_with_help_link(): # pragma: no cover

View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup from setuptools import find_packages, setup
VERSION = '1.3.21' VERSION = '1.3.22'
setup( setup(

View file

@ -1,3 +1,4 @@
import logging
import subprocess import subprocess
from flexmock import flexmock from flexmock import flexmock
@ -5,6 +6,90 @@ from flexmock import flexmock
from borgmatic.commands import borgmatic as module from borgmatic.commands import borgmatic as module
def test_run_configuration_runs_actions_for_each_repository():
flexmock(module.borg_environment).should_receive('initialize')
expected_results = [flexmock(), flexmock()]
flexmock(module).should_receive('run_actions').and_return(expected_results[:1]).and_return(
expected_results[1:]
)
config = {'location': {'repositories': ['foo', 'bar']}}
arguments = {'global': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_run_configuration_executes_hooks_for_create_action():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.hook).should_receive('execute_hook').twice()
flexmock(module).should_receive('run_actions').and_return([])
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(dry_run=False), 'create': flexmock()}
list(module.run_configuration('test.yaml', config, arguments))
def test_run_configuration_logs_actions_error():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.hook).should_receive('execute_hook')
expected_results = [flexmock()]
flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').and_raise(OSError)
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(dry_run=False)}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_run_configuration_logs_pre_hook_error():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.hook).should_receive('execute_hook').and_raise(OSError).and_return(None)
expected_results = [flexmock()]
flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').never()
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_run_configuration_logs_post_hook_error():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.hook).should_receive('execute_hook').and_return(None).and_raise(
OSError
).and_return(None)
expected_results = [flexmock()]
flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').and_return([])
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_run_configuration_logs_on_error_hook_error():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.hook).should_receive('execute_hook').and_raise(OSError)
expected_results = [flexmock(), flexmock()]
flexmock(module).should_receive('make_error_log_records').and_return(
expected_results[:1]
).and_return(expected_results[1:])
flexmock(module).should_receive('run_actions').and_raise(OSError)
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(dry_run=False)}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_load_configurations_collects_parsed_configurations(): def test_load_configurations_collects_parsed_configurations():
configuration = flexmock() configuration = flexmock()
other_configuration = flexmock() other_configuration = flexmock()
@ -24,34 +109,40 @@ def test_load_configurations_logs_critical_for_parse_error():
configs, logs = tuple(module.load_configurations(('test.yaml',))) configs, logs = tuple(module.load_configurations(('test.yaml',)))
assert configs == {} assert configs == {}
assert {log.levelno for log in logs} == {module.logging.CRITICAL} assert {log.levelno for log in logs} == {logging.CRITICAL}
def test_make_error_log_records_generates_output_logs_for_message_only():
logs = tuple(module.make_error_log_records('Error'))
assert {log.levelno for log in logs} == {logging.CRITICAL}
def test_make_error_log_records_generates_output_logs_for_called_process_error(): def test_make_error_log_records_generates_output_logs_for_called_process_error():
logs = tuple( logs = tuple(
module.make_error_log_records( module.make_error_log_records(
subprocess.CalledProcessError(1, 'ls', 'error output'), 'Error' 'Error', subprocess.CalledProcessError(1, 'ls', 'error output')
) )
) )
assert {log.levelno for log in logs} == {module.logging.CRITICAL} assert {log.levelno for log in logs} == {logging.CRITICAL}
assert any(log for log in logs if 'error output' in str(log)) assert any(log for log in logs if 'error output' in str(log))
def test_make_error_log_records_generates_logs_for_value_error(): def test_make_error_log_records_generates_logs_for_value_error():
logs = tuple(module.make_error_log_records(ValueError(), 'Error')) logs = tuple(module.make_error_log_records('Error', ValueError()))
assert {log.levelno for log in logs} == {module.logging.CRITICAL} assert {log.levelno for log in logs} == {logging.CRITICAL}
def test_make_error_log_records_generates_logs_for_os_error(): def test_make_error_log_records_generates_logs_for_os_error():
logs = tuple(module.make_error_log_records(OSError(), 'Error')) logs = tuple(module.make_error_log_records('Error', OSError()))
assert {log.levelno for log in logs} == {module.logging.CRITICAL} assert {log.levelno for log in logs} == {logging.CRITICAL}
def test_make_error_log_records_generates_nothing_for_other_error(): def test_make_error_log_records_generates_nothing_for_other_error():
logs = tuple(module.make_error_log_records(KeyError(), 'Error')) logs = tuple(module.make_error_log_records('Error', KeyError()))
assert logs == () assert logs == ()
@ -65,7 +156,7 @@ def test_collect_configuration_run_summary_logs_info_for_success():
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
) )
assert {log.levelno for log in logs} == {module.logging.INFO} assert {log.levelno for log in logs} == {logging.INFO}
def test_collect_configuration_run_summary_executes_hooks_for_create(): def test_collect_configuration_run_summary_executes_hooks_for_create():
@ -76,7 +167,7 @@ def test_collect_configuration_run_summary_executes_hooks_for_create():
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
) )
assert {log.levelno for log in logs} == {module.logging.INFO} assert {log.levelno for log in logs} == {logging.INFO}
def test_collect_configuration_run_summary_logs_info_for_success_with_extract(): def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
@ -88,56 +179,74 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
) )
assert {log.levelno for log in logs} == {module.logging.INFO} assert {log.levelno for log in logs} == {logging.INFO}
def test_collect_configuration_run_summary_logs_critical_for_extract_with_repository_error(): def test_collect_configuration_run_summary_logs_extract_with_repository_error():
flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise( flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
ValueError ValueError
) )
expected_logs = (flexmock(),)
flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
arguments = {'extract': flexmock(repository='repo')} arguments = {'extract': flexmock(repository='repo')}
logs = tuple( logs = tuple(
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
) )
assert {log.levelno for log in logs} == {module.logging.CRITICAL} assert logs == expected_logs
def test_collect_configuration_run_summary_logs_critical_for_pre_hook_error(): def test_collect_configuration_run_summary_logs_missing_configs_error():
arguments = {'global': flexmock(config_paths=[])}
expected_logs = (flexmock(),)
flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
logs = tuple(module.collect_configuration_run_summary_logs({}, arguments=arguments))
assert logs == expected_logs
def test_collect_configuration_run_summary_logs_pre_hook_error():
flexmock(module.hook).should_receive('execute_hook').and_raise(ValueError) flexmock(module.hook).should_receive('execute_hook').and_raise(ValueError)
expected_logs = (flexmock(),)
flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)} arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
logs = tuple( logs = tuple(
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
) )
assert {log.levelno for log in logs} == {module.logging.CRITICAL} assert logs == expected_logs
def test_collect_configuration_run_summary_logs_critical_for_post_hook_error(): def test_collect_configuration_run_summary_logs_post_hook_error():
flexmock(module.hook).should_receive('execute_hook').and_return(None).and_raise(ValueError) flexmock(module.hook).should_receive('execute_hook').and_return(None).and_raise(ValueError)
flexmock(module).should_receive('run_configuration').and_return([]) flexmock(module).should_receive('run_configuration').and_return([])
expected_logs = (flexmock(),)
flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)} arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
logs = tuple( logs = tuple(
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
) )
assert {log.levelno for log in logs} == {module.logging.INFO, module.logging.CRITICAL} assert expected_logs[0] in logs
def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_and_repository_error(): def test_collect_configuration_run_summary_logs_for_list_with_archive_and_repository_error():
flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise( flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
ValueError ValueError
) )
expected_logs = (flexmock(),)
flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
arguments = {'list': flexmock(repository='repo', archive='test')} arguments = {'list': flexmock(repository='repo', archive='test')}
logs = tuple( logs = tuple(
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
) )
assert {log.levelno for log in logs} == {module.logging.CRITICAL} assert logs == expected_logs
def test_collect_configuration_run_summary_logs_info_for_success_with_list(): def test_collect_configuration_run_summary_logs_info_for_success_with_list():
@ -148,25 +257,13 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_list():
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
) )
assert {log.levelno for log in logs} == {module.logging.INFO} assert {log.levelno for log in logs} == {logging.INFO}
def test_collect_configuration_run_summary_logs_critical_for_run_value_error(): def test_collect_configuration_run_summary_logs_run_configuration_error():
flexmock(module.validate).should_receive('guard_configuration_contains_repository') flexmock(module.validate).should_receive('guard_configuration_contains_repository')
flexmock(module).should_receive('run_configuration').and_raise(ValueError) flexmock(module).should_receive('run_configuration').and_return(
arguments = {} [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))]
logs = tuple(
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert {log.levelno for log in logs} == {module.logging.CRITICAL}
def test_collect_configuration_run_summary_logs_critical_including_output_for_run_process_error():
flexmock(module.validate).should_receive('guard_configuration_contains_repository')
flexmock(module).should_receive('run_configuration').and_raise(
subprocess.CalledProcessError(1, 'command', 'error output')
) )
arguments = {} arguments = {}
@ -174,8 +271,7 @@ def test_collect_configuration_run_summary_logs_critical_including_output_for_ru
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments) module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
) )
assert {log.levelno for log in logs} == {module.logging.CRITICAL} assert {log.levelno for log in logs} == {logging.CRITICAL}
assert any(log for log in logs if 'error output' in str(log))
def test_collect_configuration_run_summary_logs_outputs_merged_json_results(): def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
@ -190,12 +286,3 @@ def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
{'test.yaml': {}, 'test2.yaml': {}}, arguments=arguments {'test.yaml': {}, 'test2.yaml': {}}, arguments=arguments
) )
) )
def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
flexmock(module).should_receive('run_configuration').and_return([])
arguments = {'global': flexmock(config_paths=[])}
logs = tuple(module.collect_configuration_run_summary_logs({}, arguments=arguments))
assert {log.levelno for log in logs} == {module.logging.CRITICAL}