Backup to a removable drive or intermittent server via "soft failure" feature (#284).
This commit is contained in:
parent
fdbb2ee905
commit
2405e97c38
8 changed files with 212 additions and 3 deletions
3
NEWS
3
NEWS
|
@ -3,6 +3,9 @@
|
|||
* #277: Customize Healthchecks log level via borgmatic "--monitoring-verbosity" flag.
|
||||
* #280: Change "exclude_if_present" option to support multiple filenames that indicate a directory
|
||||
should be excluded from backups, rather than just a single filename.
|
||||
* #284: Backup to a removable drive or intermittent server via "soft failure" feature. See the
|
||||
documentation for more information:
|
||||
https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/
|
||||
* #287: View consistency check progress via "--progress" flag for "check" action.
|
||||
* For "create" and "prune" actions, no longer list files or show detailed stats at any verbosities
|
||||
by default. You can opt back in with "--files" or "--stats" flags.
|
||||
|
|
|
@ -24,7 +24,7 @@ location:
|
|||
repositories:
|
||||
- 1234@usw-s001.rsync.net:backups.borg
|
||||
- k8pDxu32@k8pDxu32.repo.borgbase.com:repo
|
||||
- /var/lib/backups/backups.borg
|
||||
- /var/lib/backups/local.borg
|
||||
|
||||
retention:
|
||||
# Retention policy for how many backups to keep.
|
||||
|
@ -80,6 +80,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
|
|||
* [Extract a backup](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/)
|
||||
* [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
|
||||
* [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
|
||||
* [Backup to a removable drive or an intermittent server](https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/)
|
||||
* [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
|
||||
* [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
|
||||
|
||||
|
|
|
@ -83,6 +83,9 @@ def run_configuration(config_filename, config, arguments):
|
|||
global_arguments.dry_run,
|
||||
)
|
||||
except (OSError, CalledProcessError) as error:
|
||||
if command.considered_soft_failure(config_filename, error):
|
||||
return
|
||||
|
||||
encountered_error = error
|
||||
yield from make_error_log_records(
|
||||
'{}: Error running pre-backup hook'.format(config_filename), error
|
||||
|
@ -138,6 +141,9 @@ def run_configuration(config_filename, config, arguments):
|
|||
global_arguments.dry_run,
|
||||
)
|
||||
except (OSError, CalledProcessError) as error:
|
||||
if command.considered_soft_failure(config_filename, error):
|
||||
return
|
||||
|
||||
encountered_error = error
|
||||
yield from make_error_log_records(
|
||||
'{}: Error running post-backup hook'.format(config_filename), error
|
||||
|
@ -165,6 +171,9 @@ def run_configuration(config_filename, config, arguments):
|
|||
global_arguments.dry_run,
|
||||
)
|
||||
except (OSError, CalledProcessError) as error:
|
||||
if command.considered_soft_failure(config_filename, error):
|
||||
return
|
||||
|
||||
yield from make_error_log_records(
|
||||
'{}: Error running on-error hook'.format(config_filename), error
|
||||
)
|
||||
|
|
|
@ -548,7 +548,8 @@ map:
|
|||
- type: str
|
||||
desc: |
|
||||
List of one or more shell commands or scripts to execute before running all
|
||||
actions (if one of them is "create"), run once before all configuration files.
|
||||
actions (if one of them is "create"). These are collected from all configuration
|
||||
files and then run once before all of them (prior to all actions).
|
||||
example:
|
||||
- echo "Starting actions."
|
||||
after_everything:
|
||||
|
@ -556,7 +557,8 @@ map:
|
|||
- type: str
|
||||
desc: |
|
||||
List of one or more shell commands or scripts to execute after running all
|
||||
actions (if one of them is "create"), run once after all configuration files.
|
||||
actions (if one of them is "create"). These are collected from all configuration
|
||||
files and then run once before all of them (prior to all actions).
|
||||
example:
|
||||
- echo "Completed actions."
|
||||
umask:
|
||||
|
|
|
@ -6,6 +6,9 @@ from borgmatic import execute
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
SOFT_FAIL_EXIT_CODE = 75
|
||||
|
||||
|
||||
def interpolate_context(command, context):
|
||||
'''
|
||||
Given a single hook command and a dict of context names/values, interpolate the values by
|
||||
|
@ -69,3 +72,24 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte
|
|||
finally:
|
||||
if original_umask:
|
||||
os.umask(original_umask)
|
||||
|
||||
|
||||
def considered_soft_failure(config_filename, error):
|
||||
'''
|
||||
Given a configuration filename and an exception object, return whether the exception object
|
||||
represents a subprocess.CalledProcessError with a return code of SOFT_FAIL_EXIT_CODE. If so,
|
||||
that indicates that the error is a "soft failure", and should not result in an error.
|
||||
'''
|
||||
exit_code = getattr(error, 'returncode', None)
|
||||
if exit_code is None:
|
||||
return False
|
||||
|
||||
if exit_code == SOFT_FAIL_EXIT_CODE:
|
||||
logger.info(
|
||||
'{}: Command hook exited with soft failure exit code ({}); skipping remaining actions'.format(
|
||||
config_filename, SOFT_FAIL_EXIT_CODE
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
|
|
@ -0,0 +1,106 @@
|
|||
---
|
||||
title: Backup to a removable drive or an intermittent server
|
||||
---
|
||||
## Occasional backups
|
||||
|
||||
A common situation is backing up to a repository that's only sometimes online.
|
||||
For instance, you might send most of your backups to the cloud, but
|
||||
occasionally you want to plug in an external hard drive or backup to your
|
||||
buddy's sometimes-online server for that extra level of redundancy.
|
||||
|
||||
But if you run borgmatic and your hard drive isn't plugged in, or your buddy's
|
||||
server is offline, then you'll get an annoying error message and the overall
|
||||
borgmatic run will fail (even if individual repositories complete just fine).
|
||||
|
||||
So what if you want borgmatic to swallow the error of a missing drive
|
||||
or an offline server, and continue trucking along? That's where the concept of
|
||||
"soft failure" come in.
|
||||
|
||||
## Soft failure command hooks
|
||||
|
||||
This feature leverages [borgmatic command
|
||||
hooks](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/),
|
||||
so first familiarize yourself with them. The idea is that you write a simple
|
||||
test in the form of a borgmatic hook to see if backups should proceed or not.
|
||||
|
||||
The way the test works is that if any of your hook commands return a special
|
||||
exit status of 75, that indicates to borgmatic that it's a temporary failure,
|
||||
and borgmatic should skip all subsequent actions for that configuration file.
|
||||
If you return any other status, then it's a standard success or error. (Zero is
|
||||
success; anything else other than 75 is an error).
|
||||
|
||||
So for instance, if you have an external drive that's only sometimes mounted,
|
||||
declare its repository in its own [separate configuration
|
||||
file](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/),
|
||||
say at `/etc/borgmatic.d/removable.yaml`:
|
||||
|
||||
```yaml
|
||||
location:
|
||||
source_directories:
|
||||
- /home
|
||||
|
||||
repositories:
|
||||
- /mnt/removable/backup.borg
|
||||
```
|
||||
|
||||
Then, write a `before_backup` hook in that same configuration file that uses
|
||||
the external `findmnt` utility to see whether the drive is mounted before
|
||||
proceeding.
|
||||
|
||||
```yaml
|
||||
hooks:
|
||||
before_backup:
|
||||
- findmnt /mnt/removable > /dev/null || exit 75
|
||||
```
|
||||
|
||||
What this does is check if the `findmnt` command errors when probing for a
|
||||
particular mount point. If it does error, then it returns exit code 75 to
|
||||
borgmatic. borgmatic logs the soft failure, skips all further actions in that
|
||||
configurable file, and proceeds onward to any other borgmatic configuration
|
||||
files you may have.
|
||||
|
||||
You can imagine a similar check for the sometimes-online server case:
|
||||
|
||||
```yaml
|
||||
location:
|
||||
source_directories:
|
||||
- /home
|
||||
|
||||
repositories:
|
||||
- me@buddys-server.org:backup.borg
|
||||
|
||||
hooks:
|
||||
before_backup:
|
||||
- ping -q -c 1 buddys-server.org > /dev/null || exit 75
|
||||
```
|
||||
|
||||
## Caveats and details
|
||||
|
||||
There are some caveats you should be aware of with this feature.
|
||||
|
||||
* You'll generally want to put a soft failure command in the `before_backup`
|
||||
hook, so as to gate whether the backup action occurs. While a soft failure is
|
||||
also supported in the `after_backup` hook, returning a soft failure there
|
||||
won't prevent any actions from occuring, because they've already occurred!
|
||||
Similiarly, you can return a soft failure from an `on_error` hook, but at
|
||||
that point it's too late to prevent the error.
|
||||
* Returning a soft failure does prevent further commands in the same hook from
|
||||
executing. So, like a standard error, it is an "early out". Unlike a standard
|
||||
error, borgmatic does not display it in angry red text or consider it a
|
||||
failure.
|
||||
* The soft failure only applies to the scope of a single borgmatic
|
||||
configuration file. So put anything that you don't want soft-failed, like
|
||||
always-online cloud backups, in separate configuration files from your
|
||||
soft-failing repositories.
|
||||
* The soft failure doesn't have to apply to a repository. You can even perform
|
||||
a test to make sure that individual source directories are mounted and
|
||||
available. Use your imagination!
|
||||
* This feature does not apply to `before_everything` or `after_everything`
|
||||
hooks.
|
||||
|
||||
## Related documentation
|
||||
|
||||
* [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
|
||||
* [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
|
||||
* [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
|
||||
* [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
|
|
@ -3,6 +3,7 @@ import subprocess
|
|||
|
||||
from flexmock import flexmock
|
||||
|
||||
import borgmatic.hooks.command
|
||||
from borgmatic.commands import borgmatic as module
|
||||
|
||||
|
||||
|
@ -93,6 +94,20 @@ def test_run_configuration_logs_pre_hook_error():
|
|||
assert results == expected_results
|
||||
|
||||
|
||||
def test_run_configuration_bails_for_pre_hook_soft_failure():
|
||||
flexmock(module.borg_environment).should_receive('initialize')
|
||||
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
|
||||
flexmock(module.command).should_receive('execute_hook').and_raise(error).and_return(None)
|
||||
flexmock(module).should_receive('make_error_log_records').never()
|
||||
flexmock(module).should_receive('run_actions').never()
|
||||
config = {'location': {'repositories': ['foo']}}
|
||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||
|
||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||
|
||||
assert results == []
|
||||
|
||||
|
||||
def test_run_configuration_logs_post_hook_error():
|
||||
flexmock(module.borg_environment).should_receive('initialize')
|
||||
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
|
||||
|
@ -110,6 +125,23 @@ def test_run_configuration_logs_post_hook_error():
|
|||
assert results == expected_results
|
||||
|
||||
|
||||
def test_run_configuration_bails_for_post_hook_soft_failure():
|
||||
flexmock(module.borg_environment).should_receive('initialize')
|
||||
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
|
||||
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
|
||||
error
|
||||
).and_return(None)
|
||||
flexmock(module.dispatch).should_receive('call_hooks')
|
||||
flexmock(module).should_receive('make_error_log_records').never()
|
||||
flexmock(module).should_receive('run_actions').and_return([])
|
||||
config = {'location': {'repositories': ['foo']}}
|
||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||
|
||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||
|
||||
assert results == []
|
||||
|
||||
|
||||
def test_run_configuration_logs_on_error_hook_error():
|
||||
flexmock(module.borg_environment).should_receive('initialize')
|
||||
flexmock(module.command).should_receive('execute_hook').and_raise(OSError)
|
||||
|
@ -126,6 +158,21 @@ def test_run_configuration_logs_on_error_hook_error():
|
|||
assert results == expected_results
|
||||
|
||||
|
||||
def test_run_configuration_bails_for_on_error_hook_soft_failure():
|
||||
flexmock(module.borg_environment).should_receive('initialize')
|
||||
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
|
||||
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(error)
|
||||
expected_results = [flexmock()]
|
||||
flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
|
||||
flexmock(module).should_receive('run_actions').and_raise(OSError)
|
||||
config = {'location': {'repositories': ['foo']}}
|
||||
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||
|
||||
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||
|
||||
assert results == expected_results
|
||||
|
||||
|
||||
def test_load_configurations_collects_parsed_configurations():
|
||||
configuration = flexmock()
|
||||
other_configuration = flexmock()
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import logging
|
||||
import subprocess
|
||||
|
||||
from flexmock import flexmock
|
||||
|
||||
|
@ -79,3 +80,19 @@ def test_execute_hook_on_error_logs_as_error():
|
|||
).once()
|
||||
|
||||
module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False)
|
||||
|
||||
|
||||
def test_considered_soft_failure_treats_soft_fail_exit_code_as_soft_fail():
|
||||
error = subprocess.CalledProcessError(module.SOFT_FAIL_EXIT_CODE, 'try again')
|
||||
|
||||
assert module.considered_soft_failure('config.yaml', error)
|
||||
|
||||
|
||||
def test_considered_soft_failure_does_not_treat_other_exit_code_as_soft_fail():
|
||||
error = subprocess.CalledProcessError(1, 'error')
|
||||
|
||||
assert not module.considered_soft_failure('config.yaml', error)
|
||||
|
||||
|
||||
def test_considered_soft_failure_does_not_treat_other_exception_type_as_soft_fail():
|
||||
assert not module.considered_soft_failure('config.yaml', Exception())
|
||||
|
|
Loading…
Reference in a new issue