User-defined hooks for global setup or cleanup that run before/after all actions. (#192).

NEWS

@@ -1,3 +1,8 @@
+1.3.21
+ * #192: User-defined hooks for global setup or cleanup that run before/after all actions. See the
+   documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/
+
 1.3.20
  * #205: More robust sample systemd service: boot delay, network dependency, lowered CPU/IO
    priority, etc.

borgmatic/commands/borgmatic.py

@@ -231,6 +231,32 @@ def load_configurations(config_filenames):
     return (configs, logs)
 
 
+def make_error_log_records(error, message):
+    '''
+    Given an exception object and error message text, yield a series of logging.LogRecord instances
+    with error summary information.
+    '''
+    try:
+        raise error
+    except CalledProcessError as error:
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        )
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output)
+        )
+        yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error))
+    except (ValueError, OSError) as error:
+        yield logging.makeLogRecord(
+            dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
+        )
+        yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error))
+    except:  # noqa: E722
+        # Raising above only as a means of determining the error type. Swallow the exception here
+        # because we don't want the exception to propagate out of this function.
+        pass
+
+
 def collect_configuration_run_summary_logs(configs, arguments):
     '''
     Given a dict of configuration filename to corresponding parsed configuration, and parsed
@@ -258,6 +284,33 @@ def collect_configuration_run_summary_logs(configs, arguments):
             )
             return
 
+    if not configs:
+        yield logging.makeLogRecord(
+            dict(
+                levelno=logging.CRITICAL,
+                levelname='CRITICAL',
+                msg='{}: No configuration files found'.format(
+                    ' '.join(arguments['global'].config_paths)
+                ),
+            )
+        )
+        return
+
+    try:
+        if 'create' in arguments:
+            for config_filename, config in configs.items():
+                hooks = config.get('hooks', {})
+                hook.execute_hook(
+                    hooks.get('before_everything'),
+                    hooks.get('umask'),
+                    config_filename,
+                    'pre-everything',
+                    arguments['global'].dry_run,
+                )
+    except (CalledProcessError, ValueError, OSError) as error:
+        yield from make_error_log_records(error, 'Error running pre-everything hook')
+        return
+
     # Execute the actions corresponding to each configuration file.
     json_results = []
     for config_filename, config in configs.items():
@@ -270,45 +323,27 @@ def collect_configuration_run_summary_logs(configs, arguments):
                     msg='{}: Successfully ran configuration file'.format(config_filename),
                 )
             )
-        except CalledProcessError as error:
+        except (CalledProcessError, ValueError, OSError) as error:
-            yield logging.makeLogRecord(
+            yield from make_error_log_records(
-                dict(
+                error, '{}: Error running configuration file'.format(config_filename)
-                    levelno=logging.CRITICAL,
-                    levelname='CRITICAL',
-                    msg='{}: Error running configuration file'.format(config_filename),
-                )
-            )
-            yield logging.makeLogRecord(
-                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output)
-            )
-            yield logging.makeLogRecord(
-                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
-            )
-        except (ValueError, OSError) as error:
-            yield logging.makeLogRecord(
-                dict(
-                    levelno=logging.CRITICAL,
-                    levelname='CRITICAL',
-                    msg='{}: Error running configuration file'.format(config_filename),
-                )
-            )
-            yield logging.makeLogRecord(
-                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
             )
 
     if json_results:
         sys.stdout.write(json.dumps(json_results))
 
-    if not configs:
+    try:
-        yield logging.makeLogRecord(
+        if 'create' in arguments:
-            dict(
+            for config_filename, config in configs.items():
-                levelno=logging.CRITICAL,
+                hooks = config.get('hooks', {})
-                levelname='CRITICAL',
+                hook.execute_hook(
-                msg='{}: No configuration files found'.format(
+                    hooks.get('after_everything'),
-                    ' '.join(arguments['global'].config_paths)
+                    hooks.get('umask'),
-                ),
+                    config_filename,
-            )
+                    'post-everything',
+                    arguments['global'].dry_run,
                 )
+    except (CalledProcessError, ValueError, OSError) as error:
+        yield from make_error_log_records(error, 'Error running post-everything hook')
 
 
 def exit_with_help_link():  # pragma: no cover

borgmatic/config/schema.yaml

@@ -337,31 +337,52 @@ map:
                 example: false
     hooks:
         desc: |
-            Shell commands or scripts to execute before and after a backup or if an error has occurred.
+            Shell commands or scripts to execute at various points during a borgmatic run.
-            IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic.
+            IMPORTANT: All provided commands and scripts are executed with user permissions of
-            Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to
+            borgmatic. Do not forget to set secure permissions on this configuration file (chmod
-            prevent potential shell injection or privilege escalation.
+            0600) as well as on any script called from a hook (chmod 0700) to prevent potential
+            shell injection or privilege escalation.
         map:
             before_backup:
                 seq:
                     - type: str
-                desc: List of one or more shell commands or scripts to execute before creating a backup.
+                desc: |
+                    List of one or more shell commands or scripts to execute before creating a
+                    backup, run once per configuration file.
                 example:
-                    - echo "Starting a backup job."
+                    - echo "Starting a backup."
             after_backup:
                 seq:
                     - type: str
-                desc: List of one or more shell commands or scripts to execute after creating a backup.
+                desc: |
+                    List of one or more shell commands or scripts to execute after creating a
+                    backup, run once per configuration file.
                 example:
-                    - echo "Backup created."
+                    - echo "Created a backup."
             on_error:
                 seq:
                     - type: str
                 desc: |
                     List of one or more shell commands or scripts to execute when an exception occurs
-                    during a backup or when running a hook.
+                    during a backup or when running a before_backup or after_backup hook.
                 example:
-                    - echo "Error while creating a backup or running a hook."
+                    - echo "Error while creating a backup or running a backup hook."
+            before_everything:
+                seq:
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute before running all
+                    actions (if one of them is "create"), run once before all configuration files.
+                example:
+                    - echo "Starting actions."
+            after_everything:
+                seq:
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute after running all
+                    actions (if one of them is "create"), run once after all configuration files.
+                example:
+                    - echo "Completed actions."
             umask:
                 type: scalar
                 desc: Umask used when executing hooks. Defaults to the umask that borgmatic is run with.

borgmatic/execute.py

@@ -53,6 +53,8 @@ def execute_command(full_command, output_log_level=logging.INFO, shell=False):
     Execute the given command (a sequence of command/argument strings) and log its output at the
     given log level. If output log level is None, instead capture and return the output. If
     shell is True, execute the command within a shell.
+
+    Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
     logger.debug(' '.join(full_command))
 

borgmatic/hook.py

@@ -13,6 +13,7 @@ def execute_hook(commands, umask, config_filename, description, dry_run):
     if this is a dry run.
 
     Raise ValueError if the umask cannot be parsed.
+    Raise subprocesses.CalledProcessError if an error occurs in a hook.
     '''
     if not commands:
         logger.debug('{}: No commands to run for {} hook'.format(config_filename, description))

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -20,22 +20,47 @@ hooks:
         - rm /to/file.sql
 ```
 
-borgmatic hooks run once per configuration file. `before_backup` hooks run
+The `before_backup` and `after_backup` hooks each run once per configuration
-prior to backups of all repositories. `after_backup` hooks run afterwards, but
+file. `before_backup` hooks run prior to backups of all repositories in a
-not if an error occurs in a previous hook or in the backups themselves.
+configuration file, right before the `create` action. `after_backup` hooks run
+afterwards, but not if an error occurs in a previous hook or in the backups
+themselves.
 
+You can also use `before_everything` and `after_everything` hooks to perform
+global setup or cleanup:
+
+```yaml
+hooks:
+    before_everything:
+        - set-up-stuff-globally
+    after_everything:
+        - clean-up-stuff-globally
+```
+
+`before_everything` hooks collected from all borgmatic configuration files run
+once before all configuration files (prior to all actions), but only if there
+is a `create` action. An error encountered during a `before_everything` hook
+causes borgmatic to exit without creating backups.
+
+`after_everything` hooks run once after all configuration files and actions,
+but only if there is a `create` action. It runs even if an error occurs during
+a backup or a backup hook, but not if an error occurs during a
+`before_everything` hook.
 
 ## Error hooks
 
 borgmatic also runs `on_error` hooks if an error occurs, either when creating
-a backup or running another hook. Here's an example configuration:
+a backup or running a backup hook. Here's an example configuration:
 
 ```yaml
 hooks:
     on_error:
-        - echo "Error while creating a backup or running a hook."
+        - echo "Error while creating a backup or running a backup hook."
 ```
 
+Note however that borgmatic does not run `on_error` hooks if an error occurs
+within a `before_everything` or `after_everything` hook.
+
 ## Hook output
 
 Any output produced by your hooks shows up both at the console and in syslog
@@ -48,7 +73,8 @@ your backups</a>.
 An important security note about hooks: borgmatic executes all hook commands
 with the user permissions of borgmatic itself. So to prevent potential shell
 injection or privilege escalation, do not forget to set secure permissions
-(`chmod 0700`) on borgmatic configuration files and scripts invoked by hooks.
+on borgmatic configuration files (`chmod 0600`) and scripts (`chmod 0700`)
+invoked by hooks.
 
 
 ## Related documentation

docs/how-to/set-up-backups.md

@@ -7,11 +7,11 @@ To get up and running, first [install
 Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at
 least version 1.1.
 
-Borgmatic consumes configurations in `/etc/borgmatic/` and `/etc/borgmatic.d/`
+By default, borgmatic looks for its configuration files in `/etc/borgmatic/`
-by default. Therefore, we show how to install borgmatic for the root user which
+and `/etc/borgmatic.d/`, where the root user typically has read access.
-will have access permissions for these locations by default.
 
-Run the following commands to download and install borgmatic:
+So, to download and install borgmatic as the root user, run the following
+commands:
 
 ```bash
 sudo pip3 install --user --upgrade borgmatic
@@ -39,6 +39,7 @@ borgmatic:
  * [OpenBSD](http://ports.su/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
  * [stand-alone binary](https://github.com/cmarquardt/borgmatic-binary)
+ * [virtualenv](https://virtualenv.pypa.io/en/stable/)
 
 
 ## Hosting providers

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.3.20'
+VERSION = '1.3.21'
 
 
 setup(

tests/unit/commands/test_borgmatic.py

@@ -24,10 +24,40 @@ def test_load_configurations_logs_critical_for_parse_error():
     configs, logs = tuple(module.load_configurations(('test.yaml',)))
 
     assert configs == {}
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+    assert {log.levelno for log in logs} == {module.logging.CRITICAL}
+
+
+def test_make_error_log_records_generates_output_logs_for_called_process_error():
+    logs = tuple(
+        module.make_error_log_records(
+            subprocess.CalledProcessError(1, 'ls', 'error output'), 'Error'
+        )
+    )
+
+    assert {log.levelno for log in logs} == {module.logging.CRITICAL}
+    assert any(log for log in logs if 'error output' in str(log))
+
+
+def test_make_error_log_records_generates_logs_for_value_error():
+    logs = tuple(module.make_error_log_records(ValueError(), 'Error'))
+
+    assert {log.levelno for log in logs} == {module.logging.CRITICAL}
+
+
+def test_make_error_log_records_generates_logs_for_os_error():
+    logs = tuple(module.make_error_log_records(OSError(), 'Error'))
+
+    assert {log.levelno for log in logs} == {module.logging.CRITICAL}
+
+
+def test_make_error_log_records_generates_nothing_for_other_error():
+    logs = tuple(module.make_error_log_records(KeyError(), 'Error'))
+
+    assert logs == ()
 
 
 def test_collect_configuration_run_summary_logs_info_for_success():
+    flexmock(module.hook).should_receive('execute_hook').never()
     flexmock(module).should_receive('run_configuration').and_return([])
     arguments = {}
 
@@ -35,7 +65,18 @@ def test_collect_configuration_run_summary_logs_info_for_success():
         module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
     )
 
-    assert all(log for log in logs if log.levelno == module.logging.INFO)
+    assert {log.levelno for log in logs} == {module.logging.INFO}
+
+
+def test_collect_configuration_run_summary_executes_hooks_for_create():
+    flexmock(module).should_receive('run_configuration').and_return([])
+    arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
+
+    assert {log.levelno for log in logs} == {module.logging.INFO}
 
 
 def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
@@ -47,7 +88,7 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
         module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
     )
 
-    assert all(log for log in logs if log.levelno == module.logging.INFO)
+    assert {log.levelno for log in logs} == {module.logging.INFO}
 
 
 def test_collect_configuration_run_summary_logs_critical_for_extract_with_repository_error():
@@ -60,7 +101,30 @@ def test_collect_configuration_run_summary_logs_critical_for_extract_with_reposi
         module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
     )
 
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+    assert {log.levelno for log in logs} == {module.logging.CRITICAL}
+
+
+def test_collect_configuration_run_summary_logs_critical_for_pre_hook_error():
+    flexmock(module.hook).should_receive('execute_hook').and_raise(ValueError)
+    arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
+
+    assert {log.levelno for log in logs} == {module.logging.CRITICAL}
+
+
+def test_collect_configuration_run_summary_logs_critical_for_post_hook_error():
+    flexmock(module.hook).should_receive('execute_hook').and_return(None).and_raise(ValueError)
+    flexmock(module).should_receive('run_configuration').and_return([])
+    arguments = {'create': flexmock(), 'global': flexmock(dry_run=False)}
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
+
+    assert {log.levelno for log in logs} == {module.logging.INFO, module.logging.CRITICAL}
 
 
 def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_and_repository_error():
@@ -73,7 +137,7 @@ def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_a
         module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
     )
 
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+    assert {log.levelno for log in logs} == {module.logging.CRITICAL}
 
 
 def test_collect_configuration_run_summary_logs_info_for_success_with_list():
@@ -84,7 +148,7 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_list():
         module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
     )
 
-    assert all(log for log in logs if log.levelno == module.logging.INFO)
+    assert {log.levelno for log in logs} == {module.logging.INFO}
 
 
 def test_collect_configuration_run_summary_logs_critical_for_run_value_error():
@@ -96,7 +160,7 @@ def test_collect_configuration_run_summary_logs_critical_for_run_value_error():
         module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
     )
 
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+    assert {log.levelno for log in logs} == {module.logging.CRITICAL}
 
 
 def test_collect_configuration_run_summary_logs_critical_including_output_for_run_process_error():
@@ -110,7 +174,7 @@ def test_collect_configuration_run_summary_logs_critical_including_output_for_ru
         module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
     )
 
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+    assert {log.levelno for log in logs} == {module.logging.CRITICAL}
     assert any(log for log in logs if 'error output' in str(log))
 
 
@@ -134,4 +198,4 @@ def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
 
     logs = tuple(module.collect_configuration_run_summary_logs({}, arguments=arguments))
 
-    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+    assert {log.levelno for log in logs} == {module.logging.CRITICAL}