From 06dc3364810151206101ff23dcd752e380c1f0ad Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 6 Aug 2020 09:59:48 -0700 Subject: [PATCH 1/7] Fix schema exclude patterns example path. --- borgmatic/config/schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index a30c60b..c0c25d0 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -123,7 +123,7 @@ map: patterns" for more details. example: - '*.pyc' - - ~/*/.cache + - /home/*/.cache - /etc/ssl exclude_from: seq: From 95c415f416de5f83c968049d71da36ec2d5b9344 Mon Sep 17 00:00:00 2001 From: Jakub Duchateau Date: Fri, 7 Aug 2020 16:41:35 +0200 Subject: [PATCH 2/7] Change "before" to "after" in desc --- borgmatic/config/schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index c0c25d0..ee947c5 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -710,7 +710,7 @@ map: List of one or more shell commands or scripts to execute after running all actions (if one of them is "create"). These are collected from all configuration files and then - run once before all of them (prior to all actions). + run once after all of them (after any action). example: - echo "Completed actions." umask: From ea1274d1c640e2fae68dc1bda2171b96a2f8e9d1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 7 Aug 2020 10:55:05 -0700 Subject: [PATCH 3/7] Remove inactive referral link. --- README.md | 8 ++++---- docs/how-to/make-backups-redundant.md | 4 ++-- docs/how-to/set-up-backups.md | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 18028d1..1f7c3a9 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ location: # Paths of local or remote repositories to backup to. repositories: - - 1234@usw-s001.rsync.net:backups.borg - k8pDxu32@k8pDxu32.repo.borgbase.com:repo + - 1234@usw-s001.rsync.net:backups.borg - /var/lib/backups/local.borg retention: @@ -66,9 +66,9 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). Healthchecks      Cronitor      Cronhub      -PagerDuty      -rsync.net      +PagerDuty BorgBase      +rsync.net      ## How-to guides @@ -101,8 +101,8 @@ services helps support borgmatic development and hosting. (These are referral links, but without any tracking scripts or cookies.)
    -
  • rsync.net: Cloud Storage provider with full support for borg and any other SSH/SFTP tool
  • BorgBase: Borg hosting service with support for monitoring, 2FA, and append-only repos
    • +
  • rsync.net: Cloud Storage provider with full support for borg and any other SSH/SFTP tool
## Support and contributing diff --git a/docs/how-to/make-backups-redundant.md b/docs/how-to/make-backups-redundant.md index cb75513..b001e32 100644 --- a/docs/how-to/make-backups-redundant.md +++ b/docs/how-to/make-backups-redundant.md @@ -16,8 +16,8 @@ location: # Paths of local or remote repositories to backup to. repositories: - - 1234@usw-s001.rsync.net:backups.borg - k8pDxu32@k8pDxu32.repo.borgbase.com:repo + - 1234@usw-s001.rsync.net:backups.borg - /var/lib/backups/local.borg ``` @@ -28,8 +28,8 @@ your source directories. Here's a way of visualizing what borgmatic does with the above configuration: -1. Backup `/home` and `/etc` to `1234@usw-s001.rsync.net:backups.borg` 2. Backup `/home` and `/etc` to `k8pDxu32@k8pDxu32.repo.borgbase.com:repo` +1. Backup `/home` and `/etc` to `1234@usw-s001.rsync.net:backups.borg` 3. Backup `/home` and `/etc` to `/var/lib/backups/local.borg` This gives you redundancy of your data across repositories and even diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index fbe347b..2f8b2b5 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -96,8 +96,8 @@ services helps support borgmatic development and hosting. (These are referral links, but without any tracking scripts or cookies.)
    -
  • rsync.net: Cloud Storage provider with full support for borg and any other SSH/SFTP tool
  • BorgBase: Borg hosting service with support for monitoring, 2FA, and append-only repos
    • +
  • rsync.net: Cloud Storage provider with full support for borg and any other SSH/SFTP tool
## Configuration From dd6ea40a36a3c93eb8c1ed39bde49c33824493ae Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Aug 2020 14:43:03 -0700 Subject: [PATCH 4/7] Clarify encryption section: It's not just for cron users. --- docs/how-to/set-up-backups.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 2f8b2b5..a3bf1b7 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -135,9 +135,8 @@ configuration](https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your ### Encryption -Note that if you plan to run borgmatic on a schedule with cron, and you -encrypt your Borg repository with a passphrase instead of a key file, you'll -either need to set the borgmatic `encryption_passphrase` configuration +If you encrypt your Borg repository with a passphrase instead of a key file, +you'll either need to set the borgmatic `encryption_passphrase` configuration variable or set the `BORG_PASSPHRASE` environment variable. See the [repository encryption section](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption) From f611fe7be31ae6231788b10cb4b8c959962a7932 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 12 Aug 2020 10:40:39 -0700 Subject: [PATCH 5/7] Clarify documentation overview of monitoring options. --- NEWS | 4 +- docs/how-to/monitor-your-backups.md | 84 ++++++++++++++++++----------- 2 files changed, 55 insertions(+), 33 deletions(-) diff --git a/NEWS b/NEWS index ba42cac..83c5e63 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ 1.5.10.dev0 * Clarify documentation on configuration overrides, specifically the portion about list syntax: - http://localhost:8080/docs/how-to/make-per-application-backups/#configuration-overrides + http://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides + * Clarify documentation overview of monitoring options: + http://torsion.org/borgmatic/docs/how-to/monitor-your-backups/ 1.5.9 * #300: Add "borgmatic export-tar" action to export an archive to a tar-formatted file or stream. diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index d53cd63..66caae1 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -10,48 +10,68 @@ and alerting comes in. There are several different ways you can monitor your backups and find out whether they're succeeding. Which of these you choose to do is up to you and -your particular infrastructure: +your particular infrastructure. -1. **Job runner alerts**: The easiest place to start is with failure alerts -from the [scheduled job -runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) (cron, -systemd, etc.) that's running borgmatic. But note that if the job doesn't even -get scheduled (e.g. due to the job runner not running), you probably won't get -an alert at all! Still, this is a decent first line of defense, especially -when combined with some of the other approaches below. -2. **borgmatic error hooks**: The `on_error` hook allows you to run an arbitrary -command or script when borgmatic itself encounters an error running your -backups. So for instance, you can run a script to send yourself a text message -alert. But note that if borgmatic doesn't actually run, this alert won't fire. -See [error +### Job runner alerts + +The easiest place to start is with failure alerts from the [scheduled job +runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) +(cron, systemd, etc.) that's running borgmatic. But note that if the job +doesn't even get scheduled (e.g. due to the job runner not running), you +probably won't get an alert at all! Still, this is a decent first line of +defense, especially when combined with some of the other approaches below. + +### Commands run on error + +The `on_error` hook allows you to run an arbitrary command or script when +borgmatic itself encounters an error running your backups. So for instance, +you can run a script to send yourself a text message alert. But note that if +borgmatic doesn't actually run, this alert won't fire. See [error hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks) below for how to configure this. -4. **borgmatic monitoring hooks**: This feature integrates with monitoring - services like [Healthchecks](https://healthchecks.io/), -[Cronitor](https://cronitor.io), [Cronhub](https://cronhub.io), and -[PagerDuty](https://www.pagerduty.com/) and pings these services whenever -borgmatic runs. That way, you'll receive an alert when something goes wrong or -(for certain hooks) the service doesn't hear from borgmatic for a configured -interval. See [Healthchecks -hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook), [Cronitor -hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook), [Cronhub -hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook), and -[PagerDuty hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook) + +### Third-party monitoring services + +borgmatic integrates with monitoring services like +[Healthchecks](https://healthchecks.io/), [Cronitor](https://cronitor.io), +[Cronhub](https://cronhub.io), and [PagerDuty](https://www.pagerduty.com/) and +pings these services whenever borgmatic runs. That way, you'll receive an +alert when something goes wrong or (for certain hooks) the service doesn't +hear from borgmatic for a configured interval. See [Healthchecks +hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook), +[Cronitor +hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook), +[Cronhub +hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook), +and [PagerDuty +hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook) below for how to configure this. -3. **Third-party monitoring software**: You can use traditional monitoring -software to consume borgmatic JSON output and track when the last -successful backup occurred. See [scripting + +While these services offer different features, you probably only need to use +one of them at most. + +### Third-party monitoring software + +You can use traditional monitoring software to consume borgmatic JSON output +and track when the last successful backup occurred. See [scripting borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic) -and [related software](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#related-software) +and [related +software](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#related-software) below for how to configure this. -5. **Borg hosting providers**: Most [Borg hosting + +### Borg hosting providers + +Most [Borg hosting providers](https://torsion.org/borgmatic/#hosting-providers) include monitoring and alerting as part of their offering. This gives you a dashboard to check on all of your backups, and can alert you if the service doesn't hear from borgmatic for a configured interval. -6. **borgmatic consistency checks**: While not strictly part of monitoring, if you -really want confidence that your backups are not only running but are -restorable as well, you can configure particular [consistency + +### Consistency checks + +While not strictly part of monitoring, if you really want confidence that your +backups are not only running but are restorable as well, you can configure +particular [consistency checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration) or even script full [extract tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/). From 72bd96c656a46bf1e2961b0e42a50041606d030b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 12 Aug 2020 11:32:00 -0700 Subject: [PATCH 6/7] Fix traceback when a configuration directory is non-readable due to directory permissions (#350). --- NEWS | 1 + borgmatic/config/collect.py | 3 +++ tests/unit/config/test_collect.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/NEWS b/NEWS index 83c5e63..12c2718 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 1.5.10.dev0 + * #350: Fix traceback when a configuration directory is non-readable due to directory permissions. * Clarify documentation on configuration overrides, specifically the portion about list syntax: http://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides * Clarify documentation overview of monitoring options: diff --git a/borgmatic/config/collect.py b/borgmatic/config/collect.py index 97c9410..a13472e 100644 --- a/borgmatic/config/collect.py +++ b/borgmatic/config/collect.py @@ -44,6 +44,9 @@ def collect_config_filenames(config_paths): yield path continue + if not os.access(path, os.R_OK): + continue + for filename in sorted(os.listdir(path)): full_filename = os.path.join(path, filename) matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml') diff --git a/tests/unit/config/test_collect.py b/tests/unit/config/test_collect.py index 4c4420a..5bab1ae 100644 --- a/tests/unit/config/test_collect.py +++ b/tests/unit/config/test_collect.py @@ -45,6 +45,7 @@ def test_collect_config_filenames_collects_yml_file_endings(): mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yml').and_return(False) + flexmock(module.os).should_receive('access').and_return(True) flexmock(module.os).should_receive('listdir') flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['foo.yml']) @@ -62,6 +63,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False) + flexmock(module.os).should_receive('access').and_return(True) flexmock(module.os).should_receive('listdir') flexmock(sys.modules['builtins']).should_receive('sorted').and_return( ['foo.yaml', 'bar', 'baz.yaml'] @@ -84,6 +86,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar.yaml~').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.txt').and_return(False) + flexmock(module.os).should_receive('access').and_return(True) flexmock(module.os).should_receive('listdir') flexmock(sys.modules['builtins']).should_receive('sorted').and_return( ['foo.yaml', 'bar.yaml~', 'baz.txt'] @@ -94,6 +97,21 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno assert config_filenames == ('/etc/borgmatic.d/foo.yaml',) +def test_collect_config_filenames_skips_permission_denied_directories(): + config_paths = ('config.yaml', '/etc/borgmatic.d') + mock_path = flexmock(module.os.path) + mock_path.should_receive('exists').and_return(True) + mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) + mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) + flexmock(module.os).should_receive('access').and_return(False) + flexmock(module.os).should_receive('listdir') + flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['config.yaml']) + + config_filenames = tuple(module.collect_config_filenames(config_paths)) + + assert config_filenames == ('config.yaml',) + + def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does_not_exist(): config_paths = ('config.yaml', '/etc/borgmatic/config.yaml') mock_path = flexmock(module.os.path) From 5356f487a579f273e7ce2460b43f41061e926548 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 12 Aug 2020 13:46:31 -0700 Subject: [PATCH 7/7] Move before/after_everything hooks so they're easier to find. --- borgmatic/config/schema.yaml | 40 ++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index ee947c5..71742b4 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -506,6 +506,26 @@ map: "check" action or an associated before/after hook. example: - echo "Error during prune/create/check." + before_everything: + seq: + - type: str + desc: | + List of one or more shell commands or scripts to execute + before running all actions (if one of them is "create"). + These are collected from all configuration files and then + run once before all of them (prior to all actions). + example: + - echo "Starting actions." + after_everything: + seq: + - type: str + desc: | + List of one or more shell commands or scripts to execute + after running all actions (if one of them is "create"). + These are collected from all configuration files and then + run once after all of them (after any action). + example: + - echo "Completed actions." postgresql_databases: seq: - map: @@ -693,26 +713,6 @@ map: documentation for details. example: https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d01 - before_everything: - seq: - - type: str - desc: | - List of one or more shell commands or scripts to execute - before running all actions (if one of them is "create"). - These are collected from all configuration files and then - run once before all of them (prior to all actions). - example: - - echo "Starting actions." - after_everything: - seq: - - type: str - desc: | - List of one or more shell commands or scripts to execute - after running all actions (if one of them is "create"). - These are collected from all configuration files and then - run once after all of them (after any action). - example: - - echo "Completed actions." umask: type: scalar desc: |