Add Healthchecks monitoring hook "ping_body_limit" option to configure how many bytes of logs to send to the Healthchecks server (#294).

This commit is contained in:
Dan Helfman 2022-05-24 12:23:38 -07:00
parent 02781662f8
commit b9741f4d0b
4 changed files with 71 additions and 6 deletions

2
NEWS
View file

@ -1,4 +1,6 @@
1.6.1.dev0
* #294: Add Healthchecks monitoring hook "ping_body_limit" option to configure how many bytes of
logs to send to the Healthchecks server.
* #402: Remove the error when "archive_name_format" is specified but a retention prefix isn't.
* #420: Warn when an unsupported variable is used in a hook command.
* #528: Improve the error message when a configuration override contains an invalid value.

View file

@ -892,6 +892,15 @@ properties:
Healthchecks ping URL or UUID to notify when a
backup begins, ends, or errors.
example: https://hc-ping.com/your-uuid-here
ping_body_limit:
type: integer
description: |
Number of bytes of borgmatic logs to send to
Healthchecks, ideally the same as PING_BODY_LIMIT
configured on the Healthchecks server. Set to 0 to
send all logs and disable this truncation. Defaults
to 100000.
example: 200000
description: |
Configuration for a monitoring integration with
Healthchecks. Create an account at https://healthchecks.io

View file

@ -13,13 +13,14 @@ MONITOR_STATE_TO_HEALTHCHECKS = {
}
PAYLOAD_TRUNCATION_INDICATOR = '...\n'
PAYLOAD_LIMIT_BYTES = 100 * 1024 - len(PAYLOAD_TRUNCATION_INDICATOR)
DEFAULT_PING_BODY_LIMIT_BYTES = 100000
class Forgetful_buffering_handler(logging.Handler):
'''
A buffering log handler that stores log messages in memory, and throws away messages (oldest
first) once a particular capacity in bytes is reached.
first) once a particular capacity in bytes is reached. But if the given byte capacity is zero,
don't throw away any messages.
'''
def __init__(self, byte_capacity, log_level):
@ -36,6 +37,9 @@ class Forgetful_buffering_handler(logging.Handler):
self.byte_count += len(message)
self.buffer.append(message)
if not self.byte_capacity:
return
while self.byte_count > self.byte_capacity and self.buffer:
self.byte_count -= len(self.buffer[0])
self.buffer.pop(0)
@ -65,15 +69,19 @@ def format_buffered_logs_for_payload():
return payload
def initialize_monitor(
hook_config, config_filename, monitoring_log_level, dry_run
): # pragma: no cover
def initialize_monitor(hook_config, config_filename, monitoring_log_level, dry_run):
'''
Add a handler to the root logger that stores in memory the most recent logs emitted. That
way, we can send them all to Healthchecks upon a finish or failure state.
'''
ping_body_limit = max(
hook_config.get('ping_body_limit', DEFAULT_PING_BODY_LIMIT_BYTES)
- len(PAYLOAD_TRUNCATION_INDICATOR),
0,
)
logging.getLogger().addHandler(
Forgetful_buffering_handler(PAYLOAD_LIMIT_BYTES, monitoring_log_level)
Forgetful_buffering_handler(ping_body_limit, monitoring_log_level)
)

View file

@ -12,6 +12,15 @@ def test_forgetful_buffering_handler_emit_collects_log_records():
assert not handler.forgot
def test_forgetful_buffering_handler_emit_collects_log_records_with_zero_byte_capacity():
handler = module.Forgetful_buffering_handler(byte_capacity=0, log_level=1)
handler.emit(flexmock(getMessage=lambda: 'foo'))
handler.emit(flexmock(getMessage=lambda: 'bar'))
assert handler.buffer == ['foo\n', 'bar\n']
assert not handler.forgot
def test_forgetful_buffering_handler_emit_forgets_log_records_when_capacity_reached():
handler = module.Forgetful_buffering_handler(byte_capacity=len('foo\nbar\n'), log_level=1)
handler.emit(flexmock(getMessage=lambda: 'foo'))
@ -60,6 +69,43 @@ def test_format_buffered_logs_for_payload_without_handler_produces_empty_payload
assert payload == ''
def test_initialize_monitor_creates_log_handler_with_ping_body_limit():
ping_body_limit = 100
monitoring_log_level = 1
flexmock(module).should_receive('Forgetful_buffering_handler').with_args(
ping_body_limit - len(module.PAYLOAD_TRUNCATION_INDICATOR), monitoring_log_level
).once()
module.initialize_monitor(
{'ping_body_limit': ping_body_limit}, 'test.yaml', monitoring_log_level, dry_run=False
)
def test_initialize_monitor_creates_log_handler_with_default_ping_body_limit():
monitoring_log_level = 1
flexmock(module).should_receive('Forgetful_buffering_handler').with_args(
module.DEFAULT_PING_BODY_LIMIT_BYTES - len(module.PAYLOAD_TRUNCATION_INDICATOR),
monitoring_log_level,
).once()
module.initialize_monitor({}, 'test.yaml', monitoring_log_level, dry_run=False)
def test_initialize_monitor_creates_log_handler_with_zero_ping_body_limit():
ping_body_limit = 0
monitoring_log_level = 1
flexmock(module).should_receive('Forgetful_buffering_handler').with_args(
ping_body_limit, monitoring_log_level
).once()
module.initialize_monitor(
{'ping_body_limit': ping_body_limit}, 'test.yaml', monitoring_log_level, dry_run=False
)
def test_ping_monitor_hits_ping_url_for_start_state():
flexmock(module).should_receive('Forgetful_buffering_handler')
hook_config = {'ping_url': 'https://example.com'}