psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Change monitoring hooks to specify the ping URL / integration key as a named option.

Browse source
This commit is contained in:
Dan Helfman 2022-05-23 20:02:10 -07:00
parent 32a1043468
commit 02781662f8
13 changed files with 199 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
NEWS
View file

@ -8,6 +8,9 @@
directory or from the directory containing the file doing the including. Previously, only the
working directory was used.
* Add a randomized delay to the sample systemd timer to spread out the load on a server.
* Change the configuration format for borgmatic monitoring hooks (Healthchecks, Cronitor,
PagerDuty, and Cronhub) to specify the ping URL / integration key as a named option. The intent
is to support additional options in the future. This change is backwards-compatible.
* Add emojis to documentation table of contents to make it easier to find particular how-to and
reference guides at a glance.

20
borgmatic/config/normalize.py
View file

@ -3,8 +3,24 @@ def normalize(config):
Given a configuration dict, apply particular hard-coded rules to normalize its contents to
adhere to the configuration schema.
'''
# Upgrade exclude_if_present from a string to a list.
exclude_if_present = config.get('location', {}).get('exclude_if_present')
# "Upgrade" exclude_if_present from a string to a list.
if isinstance(exclude_if_present, str):
config['location']['exclude_if_present'] = [exclude_if_present]
# Upgrade various monitoring hooks from a string to a dict.
healthchecks = config.get('hooks', {}).get('healthchecks')
if isinstance(healthchecks, str):
config['hooks']['healthchecks'] = {'ping_url': healthchecks}
cronitor = config.get('hooks', {}).get('cronitor')
if isinstance(cronitor, str):
config['hooks']['cronitor'] = {'ping_url': cronitor}
pagerduty = config.get('hooks', {}).get('pagerduty')
if isinstance(pagerduty, str):
config['hooks']['pagerduty'] = {'integration_key': pagerduty}
cronhub = config.get('hooks', {}).get('cronhub')
if isinstance(cronhub, str):
config['hooks']['cronhub'] = {'ping_url': cronhub}

82
borgmatic/config/schema.yaml
View file

@ -882,41 +882,69 @@ properties:
https://docs.mongodb.com/database-tools/mongorestore/ for
details.
healthchecks:
type: object
required: ['ping_url']
additionalProperties: false
properties:
ping_url:
type: string
description: |
Healthchecks ping URL or UUID to notify when a backup
begins, ends, or errors. Create an account at
https://healthchecks.io if you'd like to use this service.
See borgmatic monitoring documentation for details.
example:
https://hc-ping.com/your-uuid-here
cronitor:
type: string
Healthchecks ping URL or UUID to notify when a
backup begins, ends, or errors.
example: https://hc-ping.com/your-uuid-here
description: |
Cronitor ping URL to notify when a backup begins, ends, or
errors. Create an account at https://cronitor.io if you'd
like to use this service. See borgmatic monitoring
documentation for details.
example:
https://cronitor.link/d3x0c1
pagerduty:
type: string
description: |
PagerDuty integration key used to notify PagerDuty when a
backup errors. Create an account at
https://www.pagerduty.com/ if you'd like to use this
Configuration for a monitoring integration with
Healthchecks. Create an account at https://healthchecks.io
(or self-host Healthchecks) if you'd like to use this
service. See borgmatic monitoring documentation for details.
example:
a177cad45bd374409f78906a810a3074
cronhub:
cronitor:
type: object
required: ['ping_url']
additionalProperties: false
properties:
ping_url:
type: string
description: |
Cronhub ping URL to notify when a backup begins, ends, or
errors. Create an account at https://cronhub.io if you'd
Cronitor ping URL to notify when a backup begins,
ends, or errors.
example: https://cronitor.link/d3x0c1
description: |
Configuration for a monitoring integration with Cronitor.
Create an account at https://cronitor.io if you'd
like to use this service. See borgmatic monitoring
documentation for details.
example:
https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d01
pagerduty:
type: object
required: ['integration_key']
additionalProperties: false
properties:
integration_key:
type: string
description: |
PagerDuty integration key used to notify PagerDuty
when a backup errors.
example: a177cad45bd374409f78906a810a3074
description: |
Configuration for a monitoring integration with PagerDuty.
Create an account at https://www.pagerduty.com/ if you'd
like to use this service. See borgmatic monitoring
documentation for details.
cronhub:
type: object
required: ['ping_url']
additionalProperties: false
properties:
ping_url:
type: string
description: |
Cronhub ping URL to notify when a backup begins,
ends, or errors.
example: https://cronhub.io/ping/1f5e3410-254c-5587
description: |
Configuration for a monitoring integration with Crunhub.
Create an account at https://cronhub.io if you'd like to
use this service. See borgmatic monitoring documentation
for details.
umask:
type: integer
description: |

10
borgmatic/hooks/cronhub.py
View file

@ -22,14 +22,18 @@ def initialize_monitor(
pass
def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run):
def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
'''
Ping the given Cronhub URL, modified with the monitor.State. Use the given configuration
Ping the configured Cronhub URL, modified with the monitor.State. Use the given configuration
filename in any log entries. If this is a dry run, then don't actually ping anything.
'''
dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
formatted_state = '/{}/'.format(MONITOR_STATE_TO_CRONHUB[state])
ping_url = ping_url.replace('/start/', formatted_state).replace('/ping/', formatted_state)
ping_url = (
hook_config['ping_url']
.replace('/start/', formatted_state)
.replace('/ping/', formatted_state)
)
logger.info(
'{}: Pinging Cronhub {}{}'.format(config_filename, state.name.lower(), dry_run_label)

6
borgmatic/hooks/cronitor.py
View file

@ -22,13 +22,13 @@ def initialize_monitor(
pass
def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run):
def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
'''
Ping the given Cronitor URL, modified with the monitor.State. Use the given configuration
Ping the configured Cronitor URL, modified with the monitor.State. Use the given configuration
filename in any log entries. If this is a dry run, then don't actually ping anything.
'''
dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
ping_url = '{}/{}'.format(ping_url, MONITOR_STATE_TO_CRONITOR[state])
ping_url = '{}/{}'.format(hook_config['ping_url'], MONITOR_STATE_TO_CRONITOR[state])
logger.info(
'{}: Pinging Cronitor {}{}'.format(config_filename, state.name.lower(), dry_run_label)

14
borgmatic/hooks/healthchecks.py
View file

@ -66,7 +66,7 @@ def format_buffered_logs_for_payload():
def initialize_monitor(
ping_url_or_uuid, config_filename, monitoring_log_level, dry_run
hook_config, config_filename, monitoring_log_level, dry_run
): # pragma: no cover
'''
Add a handler to the root logger that stores in memory the most recent logs emitted. That
@ -77,16 +77,16 @@ def initialize_monitor(
)
def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level, dry_run):
def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
'''
Ping the given Healthchecks URL or UUID, modified with the monitor.State. Use the given
Ping the configured Healthchecks URL or UUID, modified with the monitor.State. Use the given
configuration filename in any log entries, and log to Healthchecks with the giving log level.
If this is a dry run, then don't actually ping anything.
'''
ping_url = (
ping_url_or_uuid
if ping_url_or_uuid.startswith('http')
else 'https://hc-ping.com/{}'.format(ping_url_or_uuid)
hook_config['ping_url']
if hook_config['ping_url'].startswith('http')
else 'https://hc-ping.com/{}'.format(hook_config['ping_url'])
)
dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
@ -109,7 +109,7 @@ def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level,
requests.post(ping_url, data=payload.encode('utf-8'))
def destroy_monitor(ping_url_or_uuid, config_filename, monitoring_log_level, dry_run):
def destroy_monitor(hook_config, config_filename, monitoring_log_level, dry_run):
'''
Remove the monitor handler that was added to the root logger. This prevents the handler from
getting reused by other instances of this monitor.

8
borgmatic/hooks/pagerduty.py
View file

@ -21,10 +21,10 @@ def initialize_monitor(
pass
def ping_monitor(integration_key, config_filename, state, monitoring_log_level, dry_run):
def ping_monitor(hook_config, config_filename, state, monitoring_log_level, dry_run):
'''
If this is an error state, create a PagerDuty event with the given integration key. Use the
given configuration filename in any log entries. If this is a dry run, then don't actually
If this is an error state, create a PagerDuty event with the configured integration key. Use
the given configuration filename in any log entries. If this is a dry run, then don't actually
create an event.
'''
if state != monitor.State.FAIL:
@ -47,7 +47,7 @@ def ping_monitor(integration_key, config_filename, state, monitoring_log_level,
)
payload = json.dumps(
{
'routing_key': integration_key,
'routing_key': hook_config['integration_key'],
'event_action': 'trigger',
'payload': {
'summary': 'backup failed on {}'.format(hostname),

12
docs/how-to/monitor-your-backups.md
View file

@ -136,7 +136,8 @@ URL" for your project. Here's an example:
```yaml
hooks:
healthchecks: https://hc-ping.com/addffa72-da17-40ae-be9c-ff591afb942a
healthchecks:
ping_url: https://hc-ping.com/addffa72-da17-40ae-be9c-ff591afb942a
```
With this hook in place, borgmatic pings your Healthchecks project when a
@ -176,7 +177,8 @@ API URL" for your monitor. Here's an example:
```yaml
hooks:
cronitor: https://cronitor.link/d3x0c1
cronitor:
ping_url: https://cronitor.link/d3x0c1
```
With this hook in place, borgmatic pings your Cronitor monitor when a backup
@ -204,7 +206,8 @@ URL" for your monitor. Here's an example:
```yaml
hooks:
cronhub: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
cronhub:
ping_url: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
```
With this hook in place, borgmatic pings your Cronhub monitor when a backup
@ -246,7 +249,8 @@ Here's an example:
```yaml
hooks:
pagerduty: a177cad45bd374409f78906a810a3074
pagerduty:
integration_key: a177cad45bd374409f78906a810a3074
```
With this hook in place, borgmatic creates a PagerDuty event for your service

16
tests/unit/config/test_normalize.py
View file

@ -19,6 +19,22 @@ from borgmatic.config import normalize as module
{'location': {'source_directories': ['foo', 'bar']}},
),
({'storage': {'compression': 'yes_please'}}, {'storage': {'compression': 'yes_please'}}),
(
{'hooks': {'healthchecks': 'https://example.com'}},
{'hooks': {'healthchecks': {'ping_url': 'https://example.com'}}},
),
(
{'hooks': {'cronitor': 'https://example.com'}},
{'hooks': {'cronitor': {'ping_url': 'https://example.com'}}},
),
(
{'hooks': {'pagerduty': 'https://example.com'}},
{'hooks': {'pagerduty': {'integration_key': 'https://example.com'}}},
),
(
{'hooks': {'cronhub': 'https://example.com'}},
{'hooks': {'cronhub': {'ping_url': 'https://example.com'}}},
),
),
)
def test_normalize_applies_hard_coded_normalization_to_config(config, expected_config):

32
tests/unit/hooks/test_cronhub.py
View file

@ -4,45 +4,57 @@ from borgmatic.hooks import cronhub as module
def test_ping_monitor_rewrites_ping_url_for_start_state():
ping_url = 'https://example.com/start/abcdef'
hook_config = {'ping_url': 'https://example.com/start/abcdef'}
flexmock(module.requests).should_receive('get').with_args('https://example.com/start/abcdef')
module.ping_monitor(
ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False
hook_config,
'config.yaml',
module.monitor.State.START,
monitoring_log_level=1,
dry_run=False,
)
def test_ping_monitor_rewrites_ping_url_and_state_for_start_state():
ping_url = 'https://example.com/ping/abcdef'
hook_config = {'ping_url': 'https://example.com/ping/abcdef'}
flexmock(module.requests).should_receive('get').with_args('https://example.com/start/abcdef')
module.ping_monitor(
ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False
hook_config,
'config.yaml',
module.monitor.State.START,
monitoring_log_level=1,
dry_run=False,
)
def test_ping_monitor_rewrites_ping_url_for_finish_state():
ping_url = 'https://example.com/start/abcdef'
hook_config = {'ping_url': 'https://example.com/start/abcdef'}
flexmock(module.requests).should_receive('get').with_args('https://example.com/finish/abcdef')
module.ping_monitor(
ping_url, 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False
hook_config,
'config.yaml',
module.monitor.State.FINISH,
monitoring_log_level=1,
dry_run=False,
)
def test_ping_monitor_rewrites_ping_url_for_fail_state():
ping_url = 'https://example.com/start/abcdef'
hook_config = {'ping_url': 'https://example.com/start/abcdef'}
flexmock(module.requests).should_receive('get').with_args('https://example.com/fail/abcdef')
module.ping_monitor(
ping_url, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
)
def test_ping_monitor_dry_run_does_not_hit_ping_url():
ping_url = 'https://example.com'
hook_config = {'ping_url': 'https://example.com'}
flexmock(module.requests).should_receive('get').never()
module.ping_monitor(
ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True
hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True
)

30
tests/unit/hooks/test_cronitor.py
View file

@ -4,36 +4,44 @@ from borgmatic.hooks import cronitor as module
def test_ping_monitor_hits_ping_url_for_start_state():
ping_url = 'https://example.com'
flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, 'run'))
hook_config = {'ping_url': 'https://example.com'}
flexmock(module.requests).should_receive('get').with_args('https://example.com/run')
module.ping_monitor(
ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False
hook_config,
'config.yaml',
module.monitor.State.START,
monitoring_log_level=1,
dry_run=False,
)
def test_ping_monitor_hits_ping_url_for_finish_state():
ping_url = 'https://example.com'
flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, 'complete'))
hook_config = {'ping_url': 'https://example.com'}
flexmock(module.requests).should_receive('get').with_args('https://example.com/complete')
module.ping_monitor(
ping_url, 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False
hook_config,
'config.yaml',
module.monitor.State.FINISH,
monitoring_log_level=1,
dry_run=False,
)
def test_ping_monitor_hits_ping_url_for_fail_state():
ping_url = 'https://example.com'
flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, 'fail'))
hook_config = {'ping_url': 'https://example.com'}
flexmock(module.requests).should_receive('get').with_args('https://example.com/fail')
module.ping_monitor(
ping_url, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
hook_config, 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
)
def test_ping_monitor_dry_run_does_not_hit_ping_url():
ping_url = 'https://example.com'
hook_config = {'ping_url': 'https://example.com'}
flexmock(module.requests).should_receive('get').never()
module.ping_monitor(
ping_url, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True
hook_config, 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=True
)

28
tests/unit/hooks/test_healthchecks.py
View file

@ -62,13 +62,13 @@ def test_format_buffered_logs_for_payload_without_handler_produces_empty_payload
def test_ping_monitor_hits_ping_url_for_start_state():
flexmock(module).should_receive('Forgetful_buffering_handler')
ping_url = 'https://example.com'
hook_config = {'ping_url': 'https://example.com'}
flexmock(module.requests).should_receive('post').with_args(
'{}/{}'.format(ping_url, 'start'), data=''.encode('utf-8')
'https://example.com/start', data=''.encode('utf-8')
)
module.ping_monitor(
ping_url,
hook_config,
'config.yaml',
state=module.monitor.State.START,
monitoring_log_level=1,
@ -77,15 +77,15 @@ def test_ping_monitor_hits_ping_url_for_start_state():
def test_ping_monitor_hits_ping_url_for_finish_state():
ping_url = 'https://example.com'
hook_config = {'ping_url': 'https://example.com'}
payload = 'data'
flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload)
flexmock(module.requests).should_receive('post').with_args(
ping_url, data=payload.encode('utf-8')
'https://example.com', data=payload.encode('utf-8')
)
module.ping_monitor(
ping_url,
hook_config,
'config.yaml',
state=module.monitor.State.FINISH,
monitoring_log_level=1,
@ -94,15 +94,15 @@ def test_ping_monitor_hits_ping_url_for_finish_state():
def test_ping_monitor_hits_ping_url_for_fail_state():
ping_url = 'https://example.com'
hook_config = {'ping_url': 'https://example.com'}
payload = 'data'
flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload)
flexmock(module.requests).should_receive('post').with_args(
'{}/{}'.format(ping_url, 'fail'), data=payload.encode('utf')
'https://example.com/fail', data=payload.encode('utf')
)
module.ping_monitor(
ping_url,
hook_config,
'config.yaml',
state=module.monitor.State.FAIL,
monitoring_log_level=1,
@ -111,15 +111,15 @@ def test_ping_monitor_hits_ping_url_for_fail_state():
def test_ping_monitor_with_ping_uuid_hits_corresponding_url():
ping_uuid = 'abcd-efgh-ijkl-mnop'
hook_config = {'ping_url': 'abcd-efgh-ijkl-mnop'}
payload = 'data'
flexmock(module).should_receive('format_buffered_logs_for_payload').and_return(payload)
flexmock(module.requests).should_receive('post').with_args(
'https://hc-ping.com/{}'.format(ping_uuid), data=payload.encode('utf-8')
'https://hc-ping.com/{}'.format(hook_config['ping_url']), data=payload.encode('utf-8')
)
module.ping_monitor(
ping_uuid,
hook_config,
'config.yaml',
state=module.monitor.State.FINISH,
monitoring_log_level=1,
@ -129,11 +129,11 @@ def test_ping_monitor_with_ping_uuid_hits_corresponding_url():
def test_ping_monitor_dry_run_does_not_hit_ping_url():
flexmock(module).should_receive('Forgetful_buffering_handler')
ping_url = 'https://example.com'
hook_config = {'ping_url': 'https://example.com'}
flexmock(module.requests).should_receive('post').never()
module.ping_monitor(
ping_url,
hook_config,
'config.yaml',
state=module.monitor.State.START,
monitoring_log_level=1,

24
tests/unit/hooks/test_pagerduty.py
View file

@ -7,7 +7,11 @@ def test_ping_monitor_ignores_start_state():
flexmock(module.requests).should_receive('post').never()
module.ping_monitor(
'abc123', 'config.yaml', module.monitor.State.START, monitoring_log_level=1, dry_run=False
{'integration_key': 'abc123'},
'config.yaml',
module.monitor.State.START,
monitoring_log_level=1,
dry_run=False,
)
@ -15,7 +19,11 @@ def test_ping_monitor_ignores_finish_state():
flexmock(module.requests).should_receive('post').never()
module.ping_monitor(
'abc123', 'config.yaml', module.monitor.State.FINISH, monitoring_log_level=1, dry_run=False
{'integration_key': 'abc123'},
'config.yaml',
module.monitor.State.FINISH,
monitoring_log_level=1,
dry_run=False,
)
@ -23,7 +31,11 @@ def test_ping_monitor_calls_api_for_fail_state():
flexmock(module.requests).should_receive('post')
module.ping_monitor(
'abc123', 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=False
{'integration_key': 'abc123'},
'config.yaml',
module.monitor.State.FAIL,
monitoring_log_level=1,
dry_run=False,
)
@ -31,5 +43,9 @@ def test_ping_monitor_dry_run_does_not_call_api():
flexmock(module.requests).should_receive('post').never()
module.ping_monitor(
'abc123', 'config.yaml', module.monitor.State.FAIL, monitoring_log_level=1, dry_run=True
{'integration_key': 'abc123'},
'config.yaml',
module.monitor.State.FAIL,
monitoring_log_level=1,
dry_run=True,
)