Monitor backups with Cronitor hook integration.

This commit is contained in:
Dan Helfman 2019-11-01 11:33:15 -07:00
parent 603f525352
commit 8fd46b8c70
9 changed files with 97 additions and 12 deletions

4
NEWS
View file

@ -1,3 +1,7 @@
1.4.3
* Monitor backups with Cronitor hook integration. See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
1.4.2 1.4.2
* Extract files to a particular directory via "borgmatic extract --destination" flag. * Extract files to a particular directory via "borgmatic extract --destination" flag.
* Rename "borgmatic extract --restore-path" flag to "--path" to reduce confusion with the separate * Rename "borgmatic extract --restore-path" flag to "--path" to reduce confusion with the separate

View file

@ -273,7 +273,7 @@ def parse_arguments(*unparsed_arguments):
'--repository', '--repository',
help='Path of repository to extract, defaults to the configured repository if there is only one', help='Path of repository to extract, defaults to the configured repository if there is only one',
) )
extract_group.add_argument('--archive', help='Name of archive to extract, required=True) extract_group.add_argument('--archive', help='Name of archive to extract', required=True)
extract_group.add_argument( extract_group.add_argument(
'--path', '--path',
'--restore-path', '--restore-path',
@ -311,7 +311,7 @@ def parse_arguments(*unparsed_arguments):
'--repository', '--repository',
help='Path of repository to restore from, defaults to the configured repository if there is only one', help='Path of repository to restore from, defaults to the configured repository if there is only one',
) )
restore_group.add_argument('--archive', help='Name of archive to restore from, required=True) restore_group.add_argument('--archive', help='Name of archive to restore from', required=True)
restore_group.add_argument( restore_group.add_argument(
'--database', '--database',
metavar='NAME', metavar='NAME',

View file

@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list
from borgmatic.borg import prune as borg_prune from borgmatic.borg import prune as borg_prune
from borgmatic.commands.arguments import parse_arguments from borgmatic.commands.arguments import parse_arguments
from borgmatic.config import checks, collect, convert, validate from borgmatic.config import checks, collect, convert, validate
from borgmatic.hooks import command, healthchecks, postgresql from borgmatic.hooks import command, cronitor, healthchecks, postgresql
from borgmatic.logger import configure_logging, should_do_markup from borgmatic.logger import configure_logging, should_do_markup
from borgmatic.signals import configure_signals from borgmatic.signals import configure_signals
from borgmatic.verbosity import verbosity_to_log_level from borgmatic.verbosity import verbosity_to_log_level
@ -56,6 +56,9 @@ def run_configuration(config_filename, config, arguments):
healthchecks.ping_healthchecks( healthchecks.ping_healthchecks(
hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start' hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start'
) )
cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run'
)
command.execute_hook( command.execute_hook(
hooks.get('before_backup'), hooks.get('before_backup'),
hooks.get('umask'), hooks.get('umask'),
@ -108,6 +111,9 @@ def run_configuration(config_filename, config, arguments):
healthchecks.ping_healthchecks( healthchecks.ping_healthchecks(
hooks.get('healthchecks'), config_filename, global_arguments.dry_run hooks.get('healthchecks'), config_filename, global_arguments.dry_run
) )
cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete'
)
except (OSError, CalledProcessError) as error: except (OSError, CalledProcessError) as error:
encountered_error = error encountered_error = error
yield from make_error_log_records( yield from make_error_log_records(
@ -129,6 +135,9 @@ def run_configuration(config_filename, config, arguments):
healthchecks.ping_healthchecks( healthchecks.ping_healthchecks(
hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail' hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail'
) )
cronitor.ping_cronitor(
hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail'
)
except (OSError, CalledProcessError) as error: except (OSError, CalledProcessError) as error:
yield from make_error_log_records( yield from make_error_log_records(
'{}: Error running on-error hook'.format(config_filename), error '{}: Error running on-error hook'.format(config_filename), error

View file

@ -430,6 +430,13 @@ map:
Create an account at https://healthchecks.io if you'd like to use this service. Create an account at https://healthchecks.io if you'd like to use this service.
example: example:
https://hc-ping.com/your-uuid-here https://hc-ping.com/your-uuid-here
cronitor:
type: str
desc: |
Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
account at https://cronitor.io if you'd like to use this service.
example:
https://cronitor.link/d3x0c1
before_everything: before_everything:
seq: seq:
- type: str - type: str

View file

@ -0,0 +1,24 @@
import logging
import requests
logger = logging.getLogger(__name__)
def ping_cronitor(ping_url, config_filename, dry_run, append):
'''
Ping the given Cronitor URL, appending the append string. Use the given configuration filename
in any log entries. If this is a dry run, then don't actually ping anything.
'''
if not ping_url:
logger.debug('{}: No Cronitor hook set'.format(config_filename))
return
dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
ping_url = '{}/{}'.format(ping_url, append)
logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label))
logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url)

View file

@ -7,12 +7,12 @@ logger = logging.getLogger(__name__)
def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None): def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
''' '''
Ping the given healthchecks.io URL or UUID, appending the append string if any. Use the given Ping the given Healthchecks URL or UUID, appending the append string if any. Use the given
configuration filename in any log entries. If this is a dry run, then don't actually ping configuration filename in any log entries. If this is a dry run, then don't actually ping
anything. anything.
''' '''
if not ping_url_or_uuid: if not ping_url_or_uuid:
logger.debug('{}: No healthchecks hook set'.format(config_filename)) logger.debug('{}: No Healthchecks hook set'.format(config_filename))
return return
ping_url = ( ping_url = (
@ -26,11 +26,11 @@ def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
ping_url = '{}/{}'.format(ping_url, append) ping_url = '{}/{}'.format(ping_url, append)
logger.info( logger.info(
'{}: Pinging healthchecks.io{}{}'.format( '{}: Pinging Healthchecks{}{}'.format(
config_filename, ' ' + append if append else '', dry_run_label config_filename, ' ' + append if append else '', dry_run_label
) )
) )
logger.debug('{}: Using healthchecks.io ping URL {}'.format(config_filename, ping_url)) logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
logging.getLogger('urllib3').setLevel(logging.ERROR) logging.getLogger('urllib3').setLevel(logging.ERROR)
requests.get(ping_url) requests.get(ping_url)

View file

@ -26,12 +26,15 @@ alert. But note that if borgmatic doesn't actually run, this alert won't fire.
See [error See [error
hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks) hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
below for how to configure this. below for how to configure this.
4. **borgmatic Healthchecks hook**: This feature integrates with the 4. **borgmatic monitoring hooks**: This feature integrates with monitoring
[Healthchecks](https://healthchecks.io/) service, and pings Healthchecks services like [Healthchecks](https://healthchecks.io/) and
whenever borgmatic runs. That way, Healthchecks can alert you when something [Cronitor](https://cronitor.io), and pings these services whenever borgmatic
goes wrong or it doesn't hear from borgmatic for a configured interval. See runs. That way, you'll receive an alert when something goes wrong or the
service doesn't hear from borgmatic for a configured interval. See
[Healthchecks [Healthchecks
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook) hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook)
and [Cronitor
hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook)
below for how to configure this. below for how to configure this.
3. **Third-party monitoring software**: You can use traditional monitoring 3. **Third-party monitoring software**: You can use traditional monitoring
software to consume borgmatic JSON output and track when the last software to consume borgmatic JSON output and track when the last
@ -115,6 +118,27 @@ mechanisms](https://healthchecks.io/#welcome-integrations) when backups fail
or it doesn't hear from borgmatic for a certain period of time. or it doesn't hear from borgmatic for a certain period of time.
## Cronitor hook
[Cronitor](https://cronitor.io/) provides "Cron monitoring and uptime healthchecks
for websites, services and APIs", and borgmatic has built-in
integration with it. Once you create a Cronitor account and cron job monitor on
their site, all you need to do is configure borgmatic with the unique "Ping
API URL" for your monitor. Here's an example:
```yaml
hooks:
cronitor: https://cronitor.link/d3x0c1
```
With this hook in place, borgmatic will ping your Cronitor monitor when a
backup begins, ends, or errors. Then you can configure Cronitor to notify you
by a [variety of
mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups
fail or it doesn't hear from borgmatic for a certain period of time.
## Scripting borgmatic ## Scripting borgmatic
To consume the output of borgmatic in other software, you can include an To consume the output of borgmatic in other software, you can include an

View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup from setuptools import find_packages, setup
VERSION = '1.4.2' VERSION = '1.4.3'
setup( setup(

View file

@ -0,0 +1,17 @@
from flexmock import flexmock
from borgmatic.hooks import cronitor as module
def test_ping_cronitor_hits_ping_url():
ping_url = 'https://example.com'
append = 'failed-so-hard'
flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
module.ping_cronitor(ping_url, 'config.yaml', dry_run=False, append=append)
def test_ping_cronitor_without_ping_url_does_not_raise():
flexmock(module.requests).should_receive('get').never()
module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops')