When a database command errors, display and log the error message instead of swallowing it (#396).
This commit is contained in:
parent
acbbd6670a
commit
d0d3a39833
6 changed files with 95 additions and 26 deletions
1
NEWS
1
NEWS
|
@ -1,4 +1,5 @@
|
||||||
1.7.10.dev0
|
1.7.10.dev0
|
||||||
|
* #396: When a database command errors, display and log the error message instead of swallowing it.
|
||||||
* #501: Optionally error if a source directory does not exist via "source_directories_must_exist"
|
* #501: Optionally error if a source directory does not exist via "source_directories_must_exist"
|
||||||
option in borgmatic's location configuration.
|
option in borgmatic's location configuration.
|
||||||
* #576: Add support for "file://" paths within "repositories" option.
|
* #576: Add support for "file://" paths within "repositories" option.
|
||||||
|
|
|
@ -43,6 +43,23 @@ def output_buffer_for_process(process, exclude_stdouts):
|
||||||
return process.stderr if process.stdout in exclude_stdouts else process.stdout
|
return process.stderr if process.stdout in exclude_stdouts else process.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def append_last_lines(last_lines, captured_output, line, output_log_level):
|
||||||
|
'''
|
||||||
|
Given a rolling list of last lines, a list of captured output, a line to append, and an output
|
||||||
|
log level, append the line to the last lines and (if necessary) the captured output. Then log
|
||||||
|
the line at the requested output log level.
|
||||||
|
'''
|
||||||
|
last_lines.append(line)
|
||||||
|
|
||||||
|
if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
|
||||||
|
last_lines.pop(0)
|
||||||
|
|
||||||
|
if output_log_level is None:
|
||||||
|
captured_output.append(line)
|
||||||
|
else:
|
||||||
|
logger.log(output_log_level, line)
|
||||||
|
|
||||||
|
|
||||||
def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
|
def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
|
||||||
'''
|
'''
|
||||||
Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
|
Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
|
||||||
|
@ -98,15 +115,12 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
|
||||||
|
|
||||||
# Keep the last few lines of output in case the process errors, and we need the output for
|
# Keep the last few lines of output in case the process errors, and we need the output for
|
||||||
# the exception below.
|
# the exception below.
|
||||||
last_lines = buffer_last_lines[ready_buffer]
|
append_last_lines(
|
||||||
last_lines.append(line)
|
buffer_last_lines[ready_buffer],
|
||||||
if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
|
captured_outputs[ready_process],
|
||||||
last_lines.pop(0)
|
line,
|
||||||
|
output_log_level,
|
||||||
if output_log_level is None:
|
)
|
||||||
captured_outputs[ready_process].append(line)
|
|
||||||
else:
|
|
||||||
logger.log(output_log_level, line)
|
|
||||||
|
|
||||||
if not still_running:
|
if not still_running:
|
||||||
break
|
break
|
||||||
|
@ -125,8 +139,18 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
|
||||||
# If an error occurs, include its output in the raised exception so that we don't
|
# If an error occurs, include its output in the raised exception so that we don't
|
||||||
# inadvertently hide error output.
|
# inadvertently hide error output.
|
||||||
output_buffer = output_buffer_for_process(process, exclude_stdouts)
|
output_buffer = output_buffer_for_process(process, exclude_stdouts)
|
||||||
|
|
||||||
last_lines = buffer_last_lines[output_buffer] if output_buffer else []
|
last_lines = buffer_last_lines[output_buffer] if output_buffer else []
|
||||||
|
|
||||||
|
# Collect any straggling output lines that came in since we last gathered output.
|
||||||
|
while output_buffer: # pragma: no cover
|
||||||
|
line = output_buffer.readline().rstrip().decode()
|
||||||
|
if not line:
|
||||||
|
break
|
||||||
|
|
||||||
|
append_last_lines(
|
||||||
|
last_lines, captured_outputs[process], line, output_log_level=logging.ERROR
|
||||||
|
)
|
||||||
|
|
||||||
if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
|
if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
|
||||||
last_lines.insert(0, '...')
|
last_lines.insert(0, '...')
|
||||||
|
|
||||||
|
|
|
@ -88,9 +88,7 @@ def execute_dump_command(
|
||||||
+ (('--user', database['username']) if 'username' in database else ())
|
+ (('--user', database['username']) if 'username' in database else ())
|
||||||
+ ('--databases',)
|
+ ('--databases',)
|
||||||
+ database_names
|
+ database_names
|
||||||
# Use shell redirection rather than execute_command(output_file=open(...)) to prevent
|
+ ('--result-file', dump_filename)
|
||||||
# the open() call on a named pipe from hanging the main borgmatic process.
|
|
||||||
+ ('>', dump_filename)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.debug(
|
logger.debug(
|
||||||
|
@ -102,7 +100,7 @@ def execute_dump_command(
|
||||||
dump.create_named_pipe_for_dump(dump_filename)
|
dump.create_named_pipe_for_dump(dump_filename)
|
||||||
|
|
||||||
return execute_command(
|
return execute_command(
|
||||||
dump_command, shell=True, extra_environment=extra_environment, run_to_completion=False,
|
dump_command, extra_environment=extra_environment, run_to_completion=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -239,7 +239,6 @@ def test_log_outputs_does_not_error_when_one_process_exits():
|
||||||
|
|
||||||
|
|
||||||
def test_log_outputs_truncates_long_error_output():
|
def test_log_outputs_truncates_long_error_output():
|
||||||
flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
|
|
||||||
flexmock(module.logger).should_receive('log')
|
flexmock(module.logger).should_receive('log')
|
||||||
flexmock(module).should_receive('command_for_process').and_return('grep')
|
flexmock(module).should_receive('command_for_process').and_return('grep')
|
||||||
|
|
||||||
|
@ -253,7 +252,7 @@ def test_log_outputs_truncates_long_error_output():
|
||||||
flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
|
flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
|
||||||
|
|
||||||
with pytest.raises(subprocess.CalledProcessError) as error:
|
with pytest.raises(subprocess.CalledProcessError) as error:
|
||||||
module.log_outputs(
|
flexmock(module, ERROR_OUTPUT_MAX_LINE_COUNT=0).log_outputs(
|
||||||
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
|
(process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -149,8 +149,7 @@ def test_execute_dump_command_runs_mysqldump():
|
||||||
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
('mysqldump', '--add-drop-database', '--databases', 'foo', '>', 'dump',),
|
('mysqldump', '--add-drop-database', '--databases', 'foo', '--result-file', 'dump',),
|
||||||
shell=True,
|
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
run_to_completion=False,
|
run_to_completion=False,
|
||||||
).and_return(process).once()
|
).and_return(process).once()
|
||||||
|
@ -176,8 +175,7 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database():
|
||||||
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
('mysqldump', '--databases', 'foo', '>', 'dump',),
|
('mysqldump', '--databases', 'foo', '--result-file', 'dump',),
|
||||||
shell=True,
|
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
run_to_completion=False,
|
run_to_completion=False,
|
||||||
).and_return(process).once()
|
).and_return(process).once()
|
||||||
|
@ -214,10 +212,9 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port():
|
||||||
'tcp',
|
'tcp',
|
||||||
'--databases',
|
'--databases',
|
||||||
'foo',
|
'foo',
|
||||||
'>',
|
'--result-file',
|
||||||
'dump',
|
'dump',
|
||||||
),
|
),
|
||||||
shell=True,
|
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
run_to_completion=False,
|
run_to_completion=False,
|
||||||
).and_return(process).once()
|
).and_return(process).once()
|
||||||
|
@ -243,8 +240,16 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password():
|
||||||
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
('mysqldump', '--add-drop-database', '--user', 'root', '--databases', 'foo', '>', 'dump',),
|
(
|
||||||
shell=True,
|
'mysqldump',
|
||||||
|
'--add-drop-database',
|
||||||
|
'--user',
|
||||||
|
'root',
|
||||||
|
'--databases',
|
||||||
|
'foo',
|
||||||
|
'--result-file',
|
||||||
|
'dump',
|
||||||
|
),
|
||||||
extra_environment={'MYSQL_PWD': 'trustsome1'},
|
extra_environment={'MYSQL_PWD': 'trustsome1'},
|
||||||
run_to_completion=False,
|
run_to_completion=False,
|
||||||
).and_return(process).once()
|
).and_return(process).once()
|
||||||
|
@ -270,8 +275,15 @@ def test_execute_dump_command_runs_mysqldump_with_options():
|
||||||
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
('mysqldump', '--stuff=such', '--add-drop-database', '--databases', 'foo', '>', 'dump',),
|
(
|
||||||
shell=True,
|
'mysqldump',
|
||||||
|
'--stuff=such',
|
||||||
|
'--add-drop-database',
|
||||||
|
'--databases',
|
||||||
|
'foo',
|
||||||
|
'--result-file',
|
||||||
|
'dump',
|
||||||
|
),
|
||||||
extra_environment=None,
|
extra_environment=None,
|
||||||
run_to_completion=False,
|
run_to_completion=False,
|
||||||
).and_return(process).once()
|
).and_return(process).once()
|
||||||
|
|
|
@ -65,6 +65,41 @@ def test_output_buffer_for_process_returns_stdout_when_not_excluded():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_append_last_lines_under_max_line_count_appends():
|
||||||
|
last_lines = ['last']
|
||||||
|
flexmock(module.logger).should_receive('log').once()
|
||||||
|
|
||||||
|
module.append_last_lines(
|
||||||
|
last_lines, captured_output=flexmock(), line='line', output_log_level=flexmock()
|
||||||
|
)
|
||||||
|
|
||||||
|
assert last_lines == ['last', 'line']
|
||||||
|
|
||||||
|
|
||||||
|
def test_append_last_lines_over_max_line_count_trims_and_appends():
|
||||||
|
original_last_lines = [str(number) for number in range(0, module.ERROR_OUTPUT_MAX_LINE_COUNT)]
|
||||||
|
last_lines = list(original_last_lines)
|
||||||
|
flexmock(module.logger).should_receive('log').once()
|
||||||
|
|
||||||
|
module.append_last_lines(
|
||||||
|
last_lines, captured_output=flexmock(), line='line', output_log_level=flexmock()
|
||||||
|
)
|
||||||
|
|
||||||
|
assert last_lines == original_last_lines[1:] + ['line']
|
||||||
|
|
||||||
|
|
||||||
|
def test_append_last_lines_with_output_log_level_none_appends_captured_output():
|
||||||
|
last_lines = ['last']
|
||||||
|
captured_output = ['captured']
|
||||||
|
flexmock(module.logger).should_receive('log').never()
|
||||||
|
|
||||||
|
module.append_last_lines(
|
||||||
|
last_lines, captured_output=captured_output, line='line', output_log_level=None
|
||||||
|
)
|
||||||
|
|
||||||
|
assert captured_output == ['captured', 'line']
|
||||||
|
|
||||||
|
|
||||||
def test_execute_command_calls_full_command():
|
def test_execute_command_calls_full_command():
|
||||||
full_command = ['foo', 'bar']
|
full_command = ['foo', 'bar']
|
||||||
flexmock(module.os, environ={'a': 'b'})
|
flexmock(module.os, environ={'a': 'b'})
|
||||||
|
|
Loading…
Reference in a new issue