Monitor backups with Cronitor hook integration.

This commit is contained in:
Dan Helfman 2019-11-01 11:33:15 -07:00
parent 603f525352
commit 8fd46b8c70
9 changed files with 97 additions and 12 deletions

4
NEWS
View file

@ -1,3 +1,7 @@
1.4.3
* Monitor backups with Cronitor hook integration. See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
1.4.2
* Extract files to a particular directory via "borgmatic extract --destination" flag.
* Rename "borgmatic extract --restore-path" flag to "--path" to reduce confusion with the separate

View file

@ -273,7 +273,7 @@ def parse_arguments(*unparsed_arguments):
'--repository',
help='Path of repository to extract, defaults to the configured repository if there is only one',
)
extract_group.add_argument('--archive', help='Name of archive to extract, required=True)
extract_group.add_argument('--archive', help='Name of archive to extract', required=True)
extract_group.add_argument(
'--path',
'--restore-path',
@ -311,7 +311,7 @@ def parse_arguments(*unparsed_arguments):
'--repository',
help='Path of repository to restore from, defaults to the configured repository if there is only one',
)
restore_group.add_argument('--archive', help='Name of archive to restore from, required=True)
restore_group.add_argument('--archive', help='Name of archive to restore from', required=True)
restore_group.add_argument(
'--database',
metavar='NAME',

View file

@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list
from borgmatic.borg import prune as borg_prune
from borgmatic.commands.arguments import parse_arguments
from borgmatic.config import checks, collect, convert, validate
from borgmatic.hooks import command, healthchecks, postgresql
from borgmatic.hooks import command, cronitor, healthchecks, postgresql
from borgmatic.logger import configure_logging, should_do_markup
from borgmatic.signals import configure_signals
from borgmatic.verbosity import verbosity_to_log_level
@ -56,6 +56,9 @@ def run_configuration(config_filename, config, arguments):
healthchecks.ping_healthchecks(
hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start'
)
cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run'
)
command.execute_hook(
hooks.get('before_backup'),
hooks.get('umask'),
@ -108,6 +111,9 @@ def run_configuration(config_filename, config, arguments):
healthchecks.ping_healthchecks(
hooks.get('healthchecks'), config_filename, global_arguments.dry_run
)
cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete'
)
except (OSError, CalledProcessError) as error:
encountered_error = error
yield from make_error_log_records(
@ -129,6 +135,9 @@ def run_configuration(config_filename, config, arguments):
healthchecks.ping_healthchecks(
hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail'
)
cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail'
)
except (OSError, CalledProcessError) as error:
yield from make_error_log_records(
'{}: Error running on-error hook'.format(config_filename), error

View file

@ -430,6 +430,13 @@ map:
Create an account at https://healthchecks.io if you'd like to use this service.
example:
https://hc-ping.com/your-uuid-here
cronitor:
type: str
desc: |
Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
account at https://cronitor.io if you'd like to use this service.
example:
https://cronitor.link/d3x0c1
before_everything:
seq:
- type: str

View file

@ -0,0 +1,24 @@
import logging
import requests
logger = logging.getLogger(__name__)
def ping_cronitor(ping_url, config_filename, dry_run, append):
'''
Ping the given Cronitor URL, appending the append string. Use the given configuration filename
in any log entries. If this is a dry run, then don't actually ping anything.
'''
if not ping_url:
logger.debug('{}: No Cronitor hook set'.format(config_filename))
return
dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
ping_url = '{}/{}'.format(ping_url, append)
logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label))
logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url)

View file

@ -7,12 +7,12 @@ logger = logging.getLogger(__name__)
def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
'''
Ping the given healthchecks.io URL or UUID, appending the append string if any. Use the given
Ping the given Healthchecks URL or UUID, appending the append string if any. Use the given
configuration filename in any log entries. If this is a dry run, then don't actually ping
anything.
'''
if not ping_url_or_uuid:
logger.debug('{}: No healthchecks hook set'.format(config_filename))
logger.debug('{}: No Healthchecks hook set'.format(config_filename))
return
ping_url = (
@ -26,11 +26,11 @@ def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
ping_url = '{}/{}'.format(ping_url, append)
logger.info(
'{}: Pinging healthchecks.io{}{}'.format(
'{}: Pinging Healthchecks{}{}'.format(
config_filename, ' ' + append if append else '', dry_run_label
)
)
logger.debug('{}: Using healthchecks.io ping URL {}'.format(config_filename, ping_url))
logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url)

View file

@ -26,12 +26,15 @@ alert. But note that if borgmatic doesn't actually run, this alert won't fire.
See [error
hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
below for how to configure this.
4. **borgmatic Healthchecks hook**: This feature integrates with the
[Healthchecks](https://healthchecks.io/) service, and pings Healthchecks
whenever borgmatic runs. That way, Healthchecks can alert you when something
goes wrong or it doesn't hear from borgmatic for a configured interval. See
4. **borgmatic monitoring hooks**: This feature integrates with monitoring
services like [Healthchecks](https://healthchecks.io/) and
[Cronitor](https://cronitor.io), and pings these services whenever borgmatic
runs. That way, you'll receive an alert when something goes wrong or the
service doesn't hear from borgmatic for a configured interval. See
[Healthchecks
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook)
and [Cronitor
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook)
below for how to configure this.
3. **Third-party monitoring software**: You can use traditional monitoring
software to consume borgmatic JSON output and track when the last
@ -115,6 +118,27 @@ mechanisms](https://healthchecks.io/#welcome-integrations) when backups fail
or it doesn't hear from borgmatic for a certain period of time.
## Cronitor hook
[Cronitor](https://cronitor.io/) provides "Cron monitoring and uptime healthchecks
for websites, services and APIs", and borgmatic has built-in
integration with it. Once you create a Cronitor account and cron job monitor on
their site, all you need to do is configure borgmatic with the unique "Ping
API URL" for your monitor. Here's an example:
```yaml
hooks:
cronitor: https://cronitor.link/d3x0c1
```
With this hook in place, borgmatic will ping your Cronitor monitor when a
backup begins, ends, or errors. Then you can configure Cronitor to notify you
by a [variety of
mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups
fail or it doesn't hear from borgmatic for a certain period of time.
## Scripting borgmatic
To consume the output of borgmatic in other software, you can include an

View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup
VERSION = '1.4.2'
VERSION = '1.4.3'
setup(

View file

@ -0,0 +1,17 @@
from flexmock import flexmock
from borgmatic.hooks import cronitor as module
def test_ping_cronitor_hits_ping_url():
ping_url = 'https://example.com'
append = 'failed-so-hard'
flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
module.ping_cronitor(ping_url, 'config.yaml', dry_run=False, append=append)
def test_ping_cronitor_without_ping_url_does_not_raise():
flexmock(module.requests).should_receive('get').never()
module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops')