From 2405e97c388aa4f6ceee28846aae8baef73fc657 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 24 Jan 2020 20:52:48 -0800 Subject: [PATCH] Backup to a removable drive or intermittent server via "soft failure" feature (#284). --- NEWS | 3 + README.md | 3 +- borgmatic/commands/borgmatic.py | 9 ++ borgmatic/config/schema.yaml | 6 +- borgmatic/hooks/command.py | 24 ++++ ...movable-drive-or-an-intermittent-server.md | 106 ++++++++++++++++++ tests/unit/commands/test_borgmatic.py | 47 ++++++++ tests/unit/hooks/test_command.py | 17 +++ 8 files changed, 212 insertions(+), 3 deletions(-) create mode 100644 docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md diff --git a/NEWS b/NEWS index 820cef5..bd15bb1 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,9 @@ * #277: Customize Healthchecks log level via borgmatic "--monitoring-verbosity" flag. * #280: Change "exclude_if_present" option to support multiple filenames that indicate a directory should be excluded from backups, rather than just a single filename. + * #284: Backup to a removable drive or intermittent server via "soft failure" feature. See the + documentation for more information: + https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/ * #287: View consistency check progress via "--progress" flag for "check" action. * For "create" and "prune" actions, no longer list files or show detailed stats at any verbosities by default. You can opt back in with "--files" or "--stats" flags. diff --git a/README.md b/README.md index 9d08811..16dc0ea 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ location: repositories: - 1234@usw-s001.rsync.net:backups.borg - k8pDxu32@k8pDxu32.repo.borgbase.com:repo - - /var/lib/backups/backups.borg + - /var/lib/backups/local.borg retention: # Retention policy for how many backups to keep. @@ -80,6 +80,7 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). * [Extract a backup](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/) * [Backup your databases](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/) + * [Backup to a removable drive or an intermittent server](https://torsion.org/borgmatic/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server/) * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/) * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 9914668..3e37776 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -83,6 +83,9 @@ def run_configuration(config_filename, config, arguments): global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: + if command.considered_soft_failure(config_filename, error): + return + encountered_error = error yield from make_error_log_records( '{}: Error running pre-backup hook'.format(config_filename), error @@ -138,6 +141,9 @@ def run_configuration(config_filename, config, arguments): global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: + if command.considered_soft_failure(config_filename, error): + return + encountered_error = error yield from make_error_log_records( '{}: Error running post-backup hook'.format(config_filename), error @@ -165,6 +171,9 @@ def run_configuration(config_filename, config, arguments): global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: + if command.considered_soft_failure(config_filename, error): + return + yield from make_error_log_records( '{}: Error running on-error hook'.format(config_filename), error ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 46fa728..5d00eeb 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -548,7 +548,8 @@ map: - type: str desc: | List of one or more shell commands or scripts to execute before running all - actions (if one of them is "create"), run once before all configuration files. + actions (if one of them is "create"). These are collected from all configuration + files and then run once before all of them (prior to all actions). example: - echo "Starting actions." after_everything: @@ -556,7 +557,8 @@ map: - type: str desc: | List of one or more shell commands or scripts to execute after running all - actions (if one of them is "create"), run once after all configuration files. + actions (if one of them is "create"). These are collected from all configuration + files and then run once before all of them (prior to all actions). example: - echo "Completed actions." umask: diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index 16dc376..aaa777a 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -6,6 +6,9 @@ from borgmatic import execute logger = logging.getLogger(__name__) +SOFT_FAIL_EXIT_CODE = 75 + + def interpolate_context(command, context): ''' Given a single hook command and a dict of context names/values, interpolate the values by @@ -69,3 +72,24 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte finally: if original_umask: os.umask(original_umask) + + +def considered_soft_failure(config_filename, error): + ''' + Given a configuration filename and an exception object, return whether the exception object + represents a subprocess.CalledProcessError with a return code of SOFT_FAIL_EXIT_CODE. If so, + that indicates that the error is a "soft failure", and should not result in an error. + ''' + exit_code = getattr(error, 'returncode', None) + if exit_code is None: + return False + + if exit_code == SOFT_FAIL_EXIT_CODE: + logger.info( + '{}: Command hook exited with soft failure exit code ({}); skipping remaining actions'.format( + config_filename, SOFT_FAIL_EXIT_CODE + ) + ) + return True + + return False diff --git a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md new file mode 100644 index 0000000..8a86022 --- /dev/null +++ b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md @@ -0,0 +1,106 @@ +--- +title: Backup to a removable drive or an intermittent server +--- +## Occasional backups + +A common situation is backing up to a repository that's only sometimes online. +For instance, you might send most of your backups to the cloud, but +occasionally you want to plug in an external hard drive or backup to your +buddy's sometimes-online server for that extra level of redundancy. + +But if you run borgmatic and your hard drive isn't plugged in, or your buddy's +server is offline, then you'll get an annoying error message and the overall +borgmatic run will fail (even if individual repositories complete just fine). + +So what if you want borgmatic to swallow the error of a missing drive +or an offline server, and continue trucking along? That's where the concept of +"soft failure" come in. + +## Soft failure command hooks + +This feature leverages [borgmatic command +hooks](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/), +so first familiarize yourself with them. The idea is that you write a simple +test in the form of a borgmatic hook to see if backups should proceed or not. + +The way the test works is that if any of your hook commands return a special +exit status of 75, that indicates to borgmatic that it's a temporary failure, +and borgmatic should skip all subsequent actions for that configuration file. +If you return any other status, then it's a standard success or error. (Zero is +success; anything else other than 75 is an error). + +So for instance, if you have an external drive that's only sometimes mounted, +declare its repository in its own [separate configuration +file](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/), +say at `/etc/borgmatic.d/removable.yaml`: + +```yaml +location: + source_directories: + - /home + + repositories: + - /mnt/removable/backup.borg +``` + +Then, write a `before_backup` hook in that same configuration file that uses +the external `findmnt` utility to see whether the drive is mounted before +proceeding. + +```yaml +hooks: + before_backup: + - findmnt /mnt/removable > /dev/null || exit 75 +``` + +What this does is check if the `findmnt` command errors when probing for a +particular mount point. If it does error, then it returns exit code 75 to +borgmatic. borgmatic logs the soft failure, skips all further actions in that +configurable file, and proceeds onward to any other borgmatic configuration +files you may have. + +You can imagine a similar check for the sometimes-online server case: + +```yaml +location: + source_directories: + - /home + + repositories: + - me@buddys-server.org:backup.borg + +hooks: + before_backup: + - ping -q -c 1 buddys-server.org > /dev/null || exit 75 +``` + +## Caveats and details + +There are some caveats you should be aware of with this feature. + + * You'll generally want to put a soft failure command in the `before_backup` + hook, so as to gate whether the backup action occurs. While a soft failure is + also supported in the `after_backup` hook, returning a soft failure there + won't prevent any actions from occuring, because they've already occurred! + Similiarly, you can return a soft failure from an `on_error` hook, but at + that point it's too late to prevent the error. + * Returning a soft failure does prevent further commands in the same hook from + executing. So, like a standard error, it is an "early out". Unlike a standard + error, borgmatic does not display it in angry red text or consider it a + failure. + * The soft failure only applies to the scope of a single borgmatic + configuration file. So put anything that you don't want soft-failed, like + always-online cloud backups, in separate configuration files from your + soft-failing repositories. + * The soft failure doesn't have to apply to a repository. You can even perform + a test to make sure that individual source directories are mounted and + available. Use your imagination! + * This feature does not apply to `before_everything` or `after_everything` + hooks. + +## Related documentation + + * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/) + * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/) + * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/) + * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 488872f..a3c020c 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -3,6 +3,7 @@ import subprocess from flexmock import flexmock +import borgmatic.hooks.command from borgmatic.commands import borgmatic as module @@ -93,6 +94,20 @@ def test_run_configuration_logs_pre_hook_error(): assert results == expected_results +def test_run_configuration_bails_for_pre_hook_soft_failure(): + flexmock(module.borg_environment).should_receive('initialize') + error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') + flexmock(module.command).should_receive('execute_hook').and_raise(error).and_return(None) + flexmock(module).should_receive('make_error_log_records').never() + flexmock(module).should_receive('run_actions').never() + config = {'location': {'repositories': ['foo']}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + + results = list(module.run_configuration('test.yaml', config, arguments)) + + assert results == [] + + def test_run_configuration_logs_post_hook_error(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise( @@ -110,6 +125,23 @@ def test_run_configuration_logs_post_hook_error(): assert results == expected_results +def test_run_configuration_bails_for_post_hook_soft_failure(): + flexmock(module.borg_environment).should_receive('initialize') + error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') + flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise( + error + ).and_return(None) + flexmock(module.dispatch).should_receive('call_hooks') + flexmock(module).should_receive('make_error_log_records').never() + flexmock(module).should_receive('run_actions').and_return([]) + config = {'location': {'repositories': ['foo']}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + + results = list(module.run_configuration('test.yaml', config, arguments)) + + assert results == [] + + def test_run_configuration_logs_on_error_hook_error(): flexmock(module.borg_environment).should_receive('initialize') flexmock(module.command).should_receive('execute_hook').and_raise(OSError) @@ -126,6 +158,21 @@ def test_run_configuration_logs_on_error_hook_error(): assert results == expected_results +def test_run_configuration_bails_for_on_error_hook_soft_failure(): + flexmock(module.borg_environment).should_receive('initialize') + error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') + flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(error) + expected_results = [flexmock()] + flexmock(module).should_receive('make_error_log_records').and_return(expected_results) + flexmock(module).should_receive('run_actions').and_raise(OSError) + config = {'location': {'repositories': ['foo']}} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + + results = list(module.run_configuration('test.yaml', config, arguments)) + + assert results == expected_results + + def test_load_configurations_collects_parsed_configurations(): configuration = flexmock() other_configuration = flexmock() diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index 8289b02..54ed706 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -1,4 +1,5 @@ import logging +import subprocess from flexmock import flexmock @@ -79,3 +80,19 @@ def test_execute_hook_on_error_logs_as_error(): ).once() module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False) + + +def test_considered_soft_failure_treats_soft_fail_exit_code_as_soft_fail(): + error = subprocess.CalledProcessError(module.SOFT_FAIL_EXIT_CODE, 'try again') + + assert module.considered_soft_failure('config.yaml', error) + + +def test_considered_soft_failure_does_not_treat_other_exit_code_as_soft_fail(): + error = subprocess.CalledProcessError(1, 'error') + + assert not module.considered_soft_failure('config.yaml', error) + + +def test_considered_soft_failure_does_not_treat_other_exception_type_as_soft_fail(): + assert not module.considered_soft_failure('config.yaml', Exception())