Backup to a removable drive or intermittent server via "soft failure" feature (#284).
This commit is contained in:
parent
fdbb2ee905
commit
2405e97c38
8 changed files with 212 additions and 3 deletions
3
NEWS
3
NEWS
|
@ -3,6 +3,9 @@
|
||||||
* #277: Customize Healthchecks log level via borgmatic "--monitoring-verbosity" flag.
|
* #277: Customize Healthchecks log level via borgmatic "--monitoring-verbosity" flag.
|
||||||
* #280: Change "exclude_if_present" option to support multiple filenames that indicate a directory
|
* #280: Change "exclude_if_present" option to support multiple filenames that indicate a directory
|
||||||
should be excluded from backups, rather than just a single filename.
|
should be excluded from backups, rather than just a single filename.
|
||||||
|
* #284: Backup to a removable drive or intermittent server via "soft failure" feature. See the
|
||||||
|
documentation for more information:
|
||||||
|
https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/
|
||||||
* #287: View consistency check progress via "--progress" flag for "check" action.
|
* #287: View consistency check progress via "--progress" flag for "check" action.
|
||||||
* For "create" and "prune" actions, no longer list files or show detailed stats at any verbosities
|
* For "create" and "prune" actions, no longer list files or show detailed stats at any verbosities
|
||||||
by default. You can opt back in with "--files" or "--stats" flags.
|
by default. You can opt back in with "--files" or "--stats" flags.
|
||||||
|
|
|
@ -24,7 +24,7 @@ location:
|
||||||
repositories:
|
repositories:
|
||||||
- 1234@usw-s001.rsync.net:backups.borg
|
- 1234@usw-s001.rsync.net:backups.borg
|
||||||
- k8pDxu32@k8pDxu32.repo.borgbase.com:repo
|
- k8pDxu32@k8pDxu32.repo.borgbase.com:repo
|
||||||
- /var/lib/backups/backups.borg
|
- /var/lib/backups/local.borg
|
||||||
|
|
||||||
retention:
|
retention:
|
||||||
# Retention policy for how many backups to keep.
|
# Retention policy for how many backups to keep.
|
||||||
|
@ -80,6 +80,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
|
||||||
* [Extract a backup](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/)
|
* [Extract a backup](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/)
|
||||||
* [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
|
* [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/)
|
||||||
* [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
|
* [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
|
||||||
|
* [Backup to a removable drive or an intermittent server](https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/)
|
||||||
* [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
|
* [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
|
||||||
* [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
|
* [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
|
||||||
|
|
||||||
|
|
|
@ -83,6 +83,9 @@ def run_configuration(config_filename, config, arguments):
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
)
|
)
|
||||||
except (OSError, CalledProcessError) as error:
|
except (OSError, CalledProcessError) as error:
|
||||||
|
if command.considered_soft_failure(config_filename, error):
|
||||||
|
return
|
||||||
|
|
||||||
encountered_error = error
|
encountered_error = error
|
||||||
yield from make_error_log_records(
|
yield from make_error_log_records(
|
||||||
'{}: Error running pre-backup hook'.format(config_filename), error
|
'{}: Error running pre-backup hook'.format(config_filename), error
|
||||||
|
@ -138,6 +141,9 @@ def run_configuration(config_filename, config, arguments):
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
)
|
)
|
||||||
except (OSError, CalledProcessError) as error:
|
except (OSError, CalledProcessError) as error:
|
||||||
|
if command.considered_soft_failure(config_filename, error):
|
||||||
|
return
|
||||||
|
|
||||||
encountered_error = error
|
encountered_error = error
|
||||||
yield from make_error_log_records(
|
yield from make_error_log_records(
|
||||||
'{}: Error running post-backup hook'.format(config_filename), error
|
'{}: Error running post-backup hook'.format(config_filename), error
|
||||||
|
@ -165,6 +171,9 @@ def run_configuration(config_filename, config, arguments):
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
)
|
)
|
||||||
except (OSError, CalledProcessError) as error:
|
except (OSError, CalledProcessError) as error:
|
||||||
|
if command.considered_soft_failure(config_filename, error):
|
||||||
|
return
|
||||||
|
|
||||||
yield from make_error_log_records(
|
yield from make_error_log_records(
|
||||||
'{}: Error running on-error hook'.format(config_filename), error
|
'{}: Error running on-error hook'.format(config_filename), error
|
||||||
)
|
)
|
||||||
|
|
|
@ -548,7 +548,8 @@ map:
|
||||||
- type: str
|
- type: str
|
||||||
desc: |
|
desc: |
|
||||||
List of one or more shell commands or scripts to execute before running all
|
List of one or more shell commands or scripts to execute before running all
|
||||||
actions (if one of them is "create"), run once before all configuration files.
|
actions (if one of them is "create"). These are collected from all configuration
|
||||||
|
files and then run once before all of them (prior to all actions).
|
||||||
example:
|
example:
|
||||||
- echo "Starting actions."
|
- echo "Starting actions."
|
||||||
after_everything:
|
after_everything:
|
||||||
|
@ -556,7 +557,8 @@ map:
|
||||||
- type: str
|
- type: str
|
||||||
desc: |
|
desc: |
|
||||||
List of one or more shell commands or scripts to execute after running all
|
List of one or more shell commands or scripts to execute after running all
|
||||||
actions (if one of them is "create"), run once after all configuration files.
|
actions (if one of them is "create"). These are collected from all configuration
|
||||||
|
files and then run once before all of them (prior to all actions).
|
||||||
example:
|
example:
|
||||||
- echo "Completed actions."
|
- echo "Completed actions."
|
||||||
umask:
|
umask:
|
||||||
|
|
|
@ -6,6 +6,9 @@ from borgmatic import execute
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
SOFT_FAIL_EXIT_CODE = 75
|
||||||
|
|
||||||
|
|
||||||
def interpolate_context(command, context):
|
def interpolate_context(command, context):
|
||||||
'''
|
'''
|
||||||
Given a single hook command and a dict of context names/values, interpolate the values by
|
Given a single hook command and a dict of context names/values, interpolate the values by
|
||||||
|
@ -69,3 +72,24 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte
|
||||||
finally:
|
finally:
|
||||||
if original_umask:
|
if original_umask:
|
||||||
os.umask(original_umask)
|
os.umask(original_umask)
|
||||||
|
|
||||||
|
|
||||||
|
def considered_soft_failure(config_filename, error):
|
||||||
|
'''
|
||||||
|
Given a configuration filename and an exception object, return whether the exception object
|
||||||
|
represents a subprocess.CalledProcessError with a return code of SOFT_FAIL_EXIT_CODE. If so,
|
||||||
|
that indicates that the error is a "soft failure", and should not result in an error.
|
||||||
|
'''
|
||||||
|
exit_code = getattr(error, 'returncode', None)
|
||||||
|
if exit_code is None:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if exit_code == SOFT_FAIL_EXIT_CODE:
|
||||||
|
logger.info(
|
||||||
|
'{}: Command hook exited with soft failure exit code ({}); skipping remaining actions'.format(
|
||||||
|
config_filename, SOFT_FAIL_EXIT_CODE
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
|
@ -0,0 +1,106 @@
|
||||||
|
---
|
||||||
|
title: Backup to a removable drive or an intermittent server
|
||||||
|
---
|
||||||
|
## Occasional backups
|
||||||
|
|
||||||
|
A common situation is backing up to a repository that's only sometimes online.
|
||||||
|
For instance, you might send most of your backups to the cloud, but
|
||||||
|
occasionally you want to plug in an external hard drive or backup to your
|
||||||
|
buddy's sometimes-online server for that extra level of redundancy.
|
||||||
|
|
||||||
|
But if you run borgmatic and your hard drive isn't plugged in, or your buddy's
|
||||||
|
server is offline, then you'll get an annoying error message and the overall
|
||||||
|
borgmatic run will fail (even if individual repositories complete just fine).
|
||||||
|
|
||||||
|
So what if you want borgmatic to swallow the error of a missing drive
|
||||||
|
or an offline server, and continue trucking along? That's where the concept of
|
||||||
|
"soft failure" come in.
|
||||||
|
|
||||||
|
## Soft failure command hooks
|
||||||
|
|
||||||
|
This feature leverages [borgmatic command
|
||||||
|
hooks](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/),
|
||||||
|
so first familiarize yourself with them. The idea is that you write a simple
|
||||||
|
test in the form of a borgmatic hook to see if backups should proceed or not.
|
||||||
|
|
||||||
|
The way the test works is that if any of your hook commands return a special
|
||||||
|
exit status of 75, that indicates to borgmatic that it's a temporary failure,
|
||||||
|
and borgmatic should skip all subsequent actions for that configuration file.
|
||||||
|
If you return any other status, then it's a standard success or error. (Zero is
|
||||||
|
success; anything else other than 75 is an error).
|
||||||
|
|
||||||
|
So for instance, if you have an external drive that's only sometimes mounted,
|
||||||
|
declare its repository in its own [separate configuration
|
||||||
|
file](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/),
|
||||||
|
say at `/etc/borgmatic.d/removable.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
location:
|
||||||
|
source_directories:
|
||||||
|
- /home
|
||||||
|
|
||||||
|
repositories:
|
||||||
|
- /mnt/removable/backup.borg
|
||||||
|
```
|
||||||
|
|
||||||
|
Then, write a `before_backup` hook in that same configuration file that uses
|
||||||
|
the external `findmnt` utility to see whether the drive is mounted before
|
||||||
|
proceeding.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
hooks:
|
||||||
|
before_backup:
|
||||||
|
- findmnt /mnt/removable > /dev/null || exit 75
|
||||||
|
```
|
||||||
|
|
||||||
|
What this does is check if the `findmnt` command errors when probing for a
|
||||||
|
particular mount point. If it does error, then it returns exit code 75 to
|
||||||
|
borgmatic. borgmatic logs the soft failure, skips all further actions in that
|
||||||
|
configurable file, and proceeds onward to any other borgmatic configuration
|
||||||
|
files you may have.
|
||||||
|
|
||||||
|
You can imagine a similar check for the sometimes-online server case:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
location:
|
||||||
|
source_directories:
|
||||||
|
- /home
|
||||||
|
|
||||||
|
repositories:
|
||||||
|
- me@buddys-server.org:backup.borg
|
||||||
|
|
||||||
|
hooks:
|
||||||
|
before_backup:
|
||||||
|
- ping -q -c 1 buddys-server.org > /dev/null || exit 75
|
||||||
|
```
|
||||||
|
|
||||||
|
## Caveats and details
|
||||||
|
|
||||||
|
There are some caveats you should be aware of with this feature.
|
||||||
|
|
||||||
|
* You'll generally want to put a soft failure command in the `before_backup`
|
||||||
|
hook, so as to gate whether the backup action occurs. While a soft failure is
|
||||||
|
also supported in the `after_backup` hook, returning a soft failure there
|
||||||
|
won't prevent any actions from occuring, because they've already occurred!
|
||||||
|
Similiarly, you can return a soft failure from an `on_error` hook, but at
|
||||||
|
that point it's too late to prevent the error.
|
||||||
|
* Returning a soft failure does prevent further commands in the same hook from
|
||||||
|
executing. So, like a standard error, it is an "early out". Unlike a standard
|
||||||
|
error, borgmatic does not display it in angry red text or consider it a
|
||||||
|
failure.
|
||||||
|
* The soft failure only applies to the scope of a single borgmatic
|
||||||
|
configuration file. So put anything that you don't want soft-failed, like
|
||||||
|
always-online cloud backups, in separate configuration files from your
|
||||||
|
soft-failing repositories.
|
||||||
|
* The soft failure doesn't have to apply to a repository. You can even perform
|
||||||
|
a test to make sure that individual source directories are mounted and
|
||||||
|
available. Use your imagination!
|
||||||
|
* This feature does not apply to `before_everything` or `after_everything`
|
||||||
|
hooks.
|
||||||
|
|
||||||
|
## Related documentation
|
||||||
|
|
||||||
|
* [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
|
||||||
|
* [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
|
||||||
|
* [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
|
||||||
|
* [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
|
|
@ -3,6 +3,7 @@ import subprocess
|
||||||
|
|
||||||
from flexmock import flexmock
|
from flexmock import flexmock
|
||||||
|
|
||||||
|
import borgmatic.hooks.command
|
||||||
from borgmatic.commands import borgmatic as module
|
from borgmatic.commands import borgmatic as module
|
||||||
|
|
||||||
|
|
||||||
|
@ -93,6 +94,20 @@ def test_run_configuration_logs_pre_hook_error():
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_configuration_bails_for_pre_hook_soft_failure():
|
||||||
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
|
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
|
||||||
|
flexmock(module.command).should_receive('execute_hook').and_raise(error).and_return(None)
|
||||||
|
flexmock(module).should_receive('make_error_log_records').never()
|
||||||
|
flexmock(module).should_receive('run_actions').never()
|
||||||
|
config = {'location': {'repositories': ['foo']}}
|
||||||
|
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||||
|
|
||||||
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
|
|
||||||
|
assert results == []
|
||||||
|
|
||||||
|
|
||||||
def test_run_configuration_logs_post_hook_error():
|
def test_run_configuration_logs_post_hook_error():
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
|
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
|
||||||
|
@ -110,6 +125,23 @@ def test_run_configuration_logs_post_hook_error():
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_configuration_bails_for_post_hook_soft_failure():
|
||||||
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
|
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
|
||||||
|
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(
|
||||||
|
error
|
||||||
|
).and_return(None)
|
||||||
|
flexmock(module.dispatch).should_receive('call_hooks')
|
||||||
|
flexmock(module).should_receive('make_error_log_records').never()
|
||||||
|
flexmock(module).should_receive('run_actions').and_return([])
|
||||||
|
config = {'location': {'repositories': ['foo']}}
|
||||||
|
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||||
|
|
||||||
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
|
|
||||||
|
assert results == []
|
||||||
|
|
||||||
|
|
||||||
def test_run_configuration_logs_on_error_hook_error():
|
def test_run_configuration_logs_on_error_hook_error():
|
||||||
flexmock(module.borg_environment).should_receive('initialize')
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
flexmock(module.command).should_receive('execute_hook').and_raise(OSError)
|
flexmock(module.command).should_receive('execute_hook').and_raise(OSError)
|
||||||
|
@ -126,6 +158,21 @@ def test_run_configuration_logs_on_error_hook_error():
|
||||||
assert results == expected_results
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_configuration_bails_for_on_error_hook_soft_failure():
|
||||||
|
flexmock(module.borg_environment).should_receive('initialize')
|
||||||
|
error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again')
|
||||||
|
flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(error)
|
||||||
|
expected_results = [flexmock()]
|
||||||
|
flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
|
||||||
|
flexmock(module).should_receive('run_actions').and_raise(OSError)
|
||||||
|
config = {'location': {'repositories': ['foo']}}
|
||||||
|
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
|
||||||
|
|
||||||
|
results = list(module.run_configuration('test.yaml', config, arguments))
|
||||||
|
|
||||||
|
assert results == expected_results
|
||||||
|
|
||||||
|
|
||||||
def test_load_configurations_collects_parsed_configurations():
|
def test_load_configurations_collects_parsed_configurations():
|
||||||
configuration = flexmock()
|
configuration = flexmock()
|
||||||
other_configuration = flexmock()
|
other_configuration = flexmock()
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import logging
|
import logging
|
||||||
|
import subprocess
|
||||||
|
|
||||||
from flexmock import flexmock
|
from flexmock import flexmock
|
||||||
|
|
||||||
|
@ -79,3 +80,19 @@ def test_execute_hook_on_error_logs_as_error():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False)
|
module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_considered_soft_failure_treats_soft_fail_exit_code_as_soft_fail():
|
||||||
|
error = subprocess.CalledProcessError(module.SOFT_FAIL_EXIT_CODE, 'try again')
|
||||||
|
|
||||||
|
assert module.considered_soft_failure('config.yaml', error)
|
||||||
|
|
||||||
|
|
||||||
|
def test_considered_soft_failure_does_not_treat_other_exit_code_as_soft_fail():
|
||||||
|
error = subprocess.CalledProcessError(1, 'error')
|
||||||
|
|
||||||
|
assert not module.considered_soft_failure('config.yaml', error)
|
||||||
|
|
||||||
|
|
||||||
|
def test_considered_soft_failure_does_not_treat_other_exception_type_as_soft_fail():
|
||||||
|
assert not module.considered_soft_failure('config.yaml', Exception())
|
||||||
|
|
Loading…
Reference in a new issue