Monitor backups with Cronhub hook integration. Fix Healthchecks/Cronitor hooks to respect dry run.

This commit is contained in:
Dan Helfman 2019-11-07 10:08:44 -08:00
parent ac777965d0
commit 17fda7281a
11 changed files with 146 additions and 16 deletions

5
NEWS
View file

@ -1,3 +1,8 @@
1.4.8
* Monitor backups with Cronhub hook integration. See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook
* Fix Healthchecks/Cronitor hooks to skip actions when the borgmatic "--dry-run" flag is used.
1.4.7 1.4.7
* #238: In documentation, clarify when Healthchecks/Cronitor hooks fire in relation to other hooks. * #238: In documentation, clarify when Healthchecks/Cronitor hooks fire in relation to other hooks.
* #239: Upgrade your borgmatic configuration to get new options and comments via * #239: Upgrade your borgmatic configuration to get new options and comments via

View file

@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list
from borgmatic.borg import prune as borg_prune from borgmatic.borg import prune as borg_prune
from borgmatic.commands.arguments import parse_arguments from borgmatic.commands.arguments import parse_arguments
from borgmatic.config import checks, collect, convert, validate from borgmatic.config import checks, collect, convert, validate
from borgmatic.hooks import command, cronitor, healthchecks, postgresql from borgmatic.hooks import command, cronhub, cronitor, healthchecks, postgresql
from borgmatic.logger import configure_logging, should_do_markup from borgmatic.logger import configure_logging, should_do_markup
from borgmatic.signals import configure_signals from borgmatic.signals import configure_signals
from borgmatic.verbosity import verbosity_to_log_level from borgmatic.verbosity import verbosity_to_log_level
@ -59,6 +59,9 @@ def run_configuration(config_filename, config, arguments):
cronitor.ping_cronitor( cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run' hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run'
) )
cronhub.ping_cronhub(
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'start'
)
command.execute_hook( command.execute_hook(
hooks.get('before_backup'), hooks.get('before_backup'),
hooks.get('umask'), hooks.get('umask'),
@ -114,6 +117,9 @@ def run_configuration(config_filename, config, arguments):
cronitor.ping_cronitor( cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete' hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete'
) )
cronhub.ping_cronhub(
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'finish'
)
except (OSError, CalledProcessError) as error: except (OSError, CalledProcessError) as error:
encountered_error = error encountered_error = error
yield from make_error_log_records( yield from make_error_log_records(
@ -138,6 +144,9 @@ def run_configuration(config_filename, config, arguments):
cronitor.ping_cronitor( cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail' hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail'
) )
cronhub.ping_cronhub(
hooks.get('cronhub'), config_filename, global_arguments.dry_run, 'fail'
)
except (OSError, CalledProcessError) as error: except (OSError, CalledProcessError) as error:
yield from make_error_log_records( yield from make_error_log_records(
'{}: Error running on-error hook'.format(config_filename), error '{}: Error running on-error hook'.format(config_filename), error

View file

@ -439,7 +439,8 @@ map:
desc: | desc: |
Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors. Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors.
Create an account at https://healthchecks.io if you'd like to use this service. Create an account at https://healthchecks.io if you'd like to use this service.
See http://localhost:8080/docs/how-to/monitor-your-backups/#healthchecks-hook See
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
for details. for details.
example: example:
https://hc-ping.com/your-uuid-here https://hc-ping.com/your-uuid-here
@ -448,10 +449,19 @@ map:
desc: | desc: |
Cronitor ping URL to notify when a backup begins, ends, or errors. Create an Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
account at https://cronitor.io if you'd like to use this service. See account at https://cronitor.io if you'd like to use this service. See
http://localhost:8080/docs/how-to/monitor-your-backups/#cronitor-hook for https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
details. for details.
example: example:
https://cronitor.link/d3x0c1 https://cronitor.link/d3x0c1
cronhub:
type: str
desc: |
Cronhub ping URL to notify when a backup begins, ends, or errors. Create an
account at https://cronhub.io if you'd like to use this service. See
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook for
details.
example:
https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
before_everything: before_everything:
seq: seq:
- type: str - type: str

View file

@ -0,0 +1,26 @@
import logging
import requests
logger = logging.getLogger(__name__)
def ping_cronhub(ping_url, config_filename, dry_run, state):
'''
Ping the given Cronhub URL, substituting in the state string. Use the given configuration
filename in any log entries. If this is a dry run, then don't actually ping anything.
'''
if not ping_url:
logger.debug('{}: No Cronhub hook set'.format(config_filename))
return
dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
formatted_state = '/{}/'.format(state)
ping_url = ping_url.replace('/start/', formatted_state).replace('/ping/', formatted_state)
logger.info('{}: Pinging Cronhub {}{}'.format(config_filename, state, dry_run_label))
logger.debug('{}: Using Cronhub ping URL {}'.format(config_filename, ping_url))
if not dry_run:
logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url)

View file

@ -20,5 +20,6 @@ def ping_cronitor(ping_url, config_filename, dry_run, append):
logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label)) logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label))
logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url)) logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
logging.getLogger('urllib3').setLevel(logging.ERROR) if not dry_run:
requests.get(ping_url) logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url)

View file

@ -32,5 +32,6 @@ def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
) )
logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url)) logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
logging.getLogger('urllib3').setLevel(logging.ERROR) if not dry_run:
requests.get(ping_url) logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url)

View file

@ -27,14 +27,15 @@ See [error
hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks) hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
below for how to configure this. below for how to configure this.
4. **borgmatic monitoring hooks**: This feature integrates with monitoring 4. **borgmatic monitoring hooks**: This feature integrates with monitoring
services like [Healthchecks](https://healthchecks.io/) and services like [Healthchecks](https://healthchecks.io/),
[Cronitor](https://cronitor.io), and pings these services whenever borgmatic [Cronitor](https://cronitor.io), and [Cronhub](https://cronhub.io), and pings
runs. That way, you'll receive an alert when something goes wrong or the these services whenever borgmatic runs. That way, you'll receive an alert when
service doesn't hear from borgmatic for a configured interval. See something goes wrong or the service doesn't hear from borgmatic for a
configured interval. See
[Healthchecks [Healthchecks
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook) hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook), [Cronitor
and [Cronitor hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook), and [Cronhub
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook) hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook)
below for how to configure this. below for how to configure this.
3. **Third-party monitoring software**: You can use traditional monitoring 3. **Third-party monitoring software**: You can use traditional monitoring
software to consume borgmatic JSON output and track when the last software to consume borgmatic JSON output and track when the last
@ -151,6 +152,37 @@ mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
or it doesn't hear from borgmatic for a certain period of time. or it doesn't hear from borgmatic for a certain period of time.
## Cronhub hook
[Cronhub](https://cronhub.io/) provides "instant alerts when any of your
background jobs fail silently or run longer than expected", and borgmatic has
built-in integration with it. Once you create a Cronhub account and monitor on
their site, all you need to do is configure borgmatic with the unique "Ping
URL" for your monitor. Here's an example:
```yaml
hooks:
cronhub: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
```
With this hook in place, borgmatic pings your Cronhub monitor when a backup
begins, ends, or errors. Specifically, before the <a
href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
hooks</a> run, borgmatic lets Cronhub know that a backup has started. Then,
if the backup completes successfully, borgmatic notifies Cronhub of the
success after the `after_backup` hooks run. And if an error occurs during the
backup, borgmatic notifies Cronhub after the `on_error` hooks run.
Note that even though you configure borgmatic with the "start" variant of the
ping URL, borgmatic substitutes the correct state into the URL when pinging
Cronhub ("start", "finish", or "fail").
You can configure Cronhub to notify you by a [variety of
mechanisms](https://docs.cronhub.io/integrations.html) when backups fail
or it doesn't hear from borgmatic for a certain period of time.
## Scripting borgmatic ## Scripting borgmatic
To consume the output of borgmatic in other software, you can include an To consume the output of borgmatic in other software, you can include an

View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup from setuptools import find_packages, setup
VERSION = '1.4.7' VERSION = '1.4.8'
setup( setup(

View file

@ -0,0 +1,32 @@
from flexmock import flexmock
from borgmatic.hooks import cronhub as module
def test_ping_cronhub_hits_ping_url_with_start_state():
ping_url = 'https://example.com/start/abcdef'
state = 'bork'
flexmock(module.requests).should_receive('get').with_args('https://example.com/bork/abcdef')
module.ping_cronhub(ping_url, 'config.yaml', dry_run=False, state=state)
def test_ping_cronhub_hits_ping_url_with_ping_state():
ping_url = 'https://example.com/ping/abcdef'
state = 'bork'
flexmock(module.requests).should_receive('get').with_args('https://example.com/bork/abcdef')
module.ping_cronhub(ping_url, 'config.yaml', dry_run=False, state=state)
def test_ping_cronhub_without_ping_url_does_not_raise():
flexmock(module.requests).should_receive('get').never()
module.ping_cronhub(ping_url=None, config_filename='config.yaml', dry_run=False, state='oops')
def test_ping_cronhub_dry_run_does_not_hit_ping_url():
ping_url = 'https://example.com'
flexmock(module.requests).should_receive('get').never()
module.ping_cronhub(ping_url, 'config.yaml', dry_run=True, state='yay')

View file

@ -15,3 +15,10 @@ def test_ping_cronitor_without_ping_url_does_not_raise():
flexmock(module.requests).should_receive('get').never() flexmock(module.requests).should_receive('get').never()
module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops') module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops')
def test_ping_cronitor_dry_run_does_not_hit_ping_url():
ping_url = 'https://example.com'
flexmock(module.requests).should_receive('get').never()
module.ping_cronitor(ping_url, 'config.yaml', dry_run=True, append='yay')

View file

@ -31,3 +31,10 @@ def test_ping_healthchecks_hits_ping_url_with_append():
flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append)) flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
module.ping_healthchecks(ping_url, 'config.yaml', dry_run=False, append=append) module.ping_healthchecks(ping_url, 'config.yaml', dry_run=False, append=append)
def test_ping_healthchecks_dry_run_does_not_hit_ping_url():
ping_url = 'https://example.com'
flexmock(module.requests).should_receive('get').never()
module.ping_healthchecks(ping_url, 'config.yaml', dry_run=True)