Add "before_actions" and "after_actions" command hooks that run before/after all the actions for each repository, update docs to cover per-repository configurations (#463).

NEWS

@@ -1,4 +1,9 @@
 1.7.0.dev0
+ * #463: Add "before_actions" and "after_actions" command hooks that run before/after all the
+   actions for each repository. These new hooks are a good place to run per-repository steps like
+   mounting/unmounting a remote filesystem.
+ * #463: Update documentation to cover per-repository configurations:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/
  * #557: Support for Borg 2 while still working with Borg 1. This includes new borgmatic actions
    like "rcreate" (replaces "init"), "rlist" (list archives in repository), "rinfo" (show repository
    info), and "transfer" (for upgrading Borg repositories). For the most part, borgmatic tries to

borgmatic/commands/borgmatic.py

@@ -252,6 +252,15 @@ def run_actions(
         'repositories': ','.join(location['repositories']),
     }
 
+    command.execute_hook(
+        hooks.get('before_actions'),
+        hooks.get('umask'),
+        config_filename,
+        'pre-actions',
+        global_arguments.dry_run,
+        **hook_context,
+    )
+
     if 'rcreate' in arguments:
         logger.info('{}: Creating repository'.format(repository))
         borg_rcreate.create_repository(
@@ -745,6 +754,15 @@ def run_actions(
                 remote_path=remote_path,
             )
 
+    command.execute_hook(
+        hooks.get('after_actions'),
+        hooks.get('umask'),
+        config_filename,
+        'post-actions',
+        global_arguments.dry_run,
+        **hook_context,
+    )
+
 
 def load_configurations(config_filenames, overrides=None, resolve_env=True):
     '''

borgmatic/config/schema.yaml

@@ -145,10 +145,10 @@ properties:
                     type: string
                 description: |
                     Any paths matching these patterns are excluded from backups.
-                    Globs and tildes are expanded. (Note however that a glob
+                    Globs and tildes are expanded. Note that a glob pattern must
-                    pattern must either start with a glob or be an absolute
+                    either start with a glob or be an absolute path. Do not
-                    path.) Do not backslash spaces in path names. See the output
+                    backslash spaces in path names. See the output of "borg help
-                    of "borg help patterns" for more details.
+                    patterns" for more details.
                 example:
                     - '*.pyc'
                     - /home/*/.cache
@@ -538,13 +538,22 @@ properties:
             prevent potential shell injection or privilege escalation.
         additionalProperties: false
         properties:
+            before_actions:
+                type: array
+                items:
+                    type: string
+                description: |
+                    List of one or more shell commands or scripts to execute
+                    before all the actions for each repository.
+                example:
+                    - echo "Starting actions."
             before_backup:
                 type: array
                 items:
                     type: string
                 description: |
                     List of one or more shell commands or scripts to execute
-                    before creating a backup, run once per configuration file.
+                    before creating a backup, run once per repository.
                 example:
                     - echo "Starting a backup."
             before_prune:
@@ -553,7 +562,7 @@ properties:
                     type: string
                 description: |
                     List of one or more shell commands or scripts to execute
-                    before pruning, run once per configuration file.
+                    before pruning, run once per repository.
                 example:
                     - echo "Starting pruning."
             before_compact:
@@ -562,7 +571,7 @@ properties:
                     type: string
                 description: |
                     List of one or more shell commands or scripts to execute
-                    before compaction, run once per configuration file.
+                    before compaction, run once per repository.
                 example:
                     - echo "Starting compaction."
             before_check:
@@ -571,7 +580,7 @@ properties:
                     type: string
                 description: |
                     List of one or more shell commands or scripts to execute
-                    before consistency checks, run once per configuration file.
+                    before consistency checks, run once per repository.
                 example:
                     - echo "Starting checks."
             before_extract:
@@ -580,7 +589,7 @@ properties:
                     type: string
                 description: |
                     List of one or more shell commands or scripts to execute
-                    before extracting a backup, run once per configuration file.
+                    before extracting a backup, run once per repository.
                 example:
                     - echo "Starting extracting."
             after_backup:
@@ -589,7 +598,7 @@ properties:
                     type: string
                 description: |
                     List of one or more shell commands or scripts to execute
-                    after creating a backup, run once per configuration file.
+                    after creating a backup, run once per repository.
                 example:
                     - echo "Finished a backup."
             after_compact:
@@ -598,7 +607,7 @@ properties:
                     type: string
                 description: |
                     List of one or more shell commands or scripts to execute
-                    after compaction, run once per configuration file.
+                    after compaction, run once per repository.
                 example:
                     - echo "Finished compaction."
             after_prune:
@@ -607,7 +616,7 @@ properties:
                     type: string
                 description: |
                     List of one or more shell commands or scripts to execute
-                    after pruning, run once per configuration file.
+                    after pruning, run once per repository.
                 example:
                     - echo "Finished pruning."
             after_check:
@@ -616,7 +625,7 @@ properties:
                     type: string
                 description: |
                     List of one or more shell commands or scripts to execute
-                    after consistency checks, run once per configuration file.
+                    after consistency checks, run once per repository.
                 example:
                     - echo "Finished checks."
             after_extract:
@@ -625,9 +634,18 @@ properties:
                     type: string
                 description: |
                     List of one or more shell commands or scripts to execute
-                    after extracting a backup, run once per configuration file.
+                    after extracting a backup, run once per repository.
                 example:
                     - echo "Finished extracting."
+            after_actions:
+                type: array
+                items:
+                    type: string
+                description: |
+                    List of one or more shell commands or scripts to execute
+                    after all actions for each repository.
+                example:
+                    - echo "Finished actions."
             on_error:
                 type: array
                 items:

docs/Dockerfile

@@ -4,7 +4,7 @@ COPY . /app
 RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
-    && for action in rcreate prune compact create check extract export-tar mount umount restore rlist list rinfo info borg; do \
+    && for action in rcreate transfer prune compact create check extract export-tar mount umount restore rlist list rinfo info borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -40,6 +40,13 @@ There are additional hooks that run before/after other actions as well. For
 instance, `before_prune` runs before a `prune` action for a repository, while
 `after_prune` runs after it.
 
+<span class="minilink minilink-addedin">New in version 1.7.0</span> The
+`before_actions` and `after_actions` hooks run before/after all the actions
+(like `create`, `prune`, etc.) for each repository. These hooks are a good
+place to run per-repository steps like mounting/unmounting a remote
+filesystem.
+
+
 ## Variable interpolation
 
 The before and after action hooks support interpolating particular runtime

docs/how-to/deal-with-very-large-backups.md

@@ -96,12 +96,12 @@ Unlike a real scheduler like cron, borgmatic only makes a best effort to run
 checks on the configured frequency. It compares that frequency with how long
 it's been since the last check for a given repository (as recorded in a file
 within `~/.borgmatic/checks`). If it hasn't been long enough, the check is
-skipped. And you still have to run `borgmatic check` (or just `borgmatic`) in
+skipped. And you still have to run `borgmatic check` (or `borgmatic` without
-order for checks to run, even when a `frequency` is configured!
+actions) in order for checks to run, even when a `frequency` is configured!
 
 This also applies *across* configuration files that have the same repository
-configured. Just make sure you have the same check frequency configured in
+configured. Make sure you have the same check frequency configured in each
-each—or the most frequently configured check will apply.
+though—or the most frequently configured check will apply.
 
 If you want to temporarily ignore your configured frequencies, you can invoke
 `borgmatic check --force` to run checks unconditionally.

docs/how-to/make-per-application-backups.md

@@ -8,13 +8,15 @@ eleventyNavigation:
 ## Multiple backup configurations
 
 You may find yourself wanting to create different backup policies for
-different applications on your system. For instance, you may want one backup
+different applications on your system or even for different backup
-configuration for your database data directory, and a different configuration
+repositories. For instance, you might want one backup configuration for your
-for your user home directories.
+database data directory and a different configuration for your user home
+directories. Or one backup configuration for your local backups with a
+different configuration for your remote repository.
 
 The way to accomplish that is pretty simple: Create multiple separate
 configuration files and place each one in a `/etc/borgmatic.d/` directory. For
-instance:
+instance, for applications:
 
 ```bash
 sudo mkdir /etc/borgmatic.d
@@ -22,6 +24,14 @@ sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
 sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
 ```
 
+Or, for repositories:
+
+```bash
+sudo mkdir /etc/borgmatic.d
+sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo1.yaml
+sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo2.yaml
+```
+
 When you set up multiple configuration files like this, borgmatic will run
 each one in turn from a single borgmatic invocation. This includes, by
 default, the traditional `/etc/borgmatic/config.yaml` as well.
@@ -29,7 +39,8 @@ default, the traditional `/etc/borgmatic/config.yaml` as well.
 Each configuration file is interpreted independently, as if you ran borgmatic
 for each configuration file one at a time. In other words, borgmatic does not
 perform any merging of configuration files by default. If you'd like borgmatic
-to merge your configuration files, see below about configuration includes.
+to merge your configuration files, for instance to avoid duplication of
+settings, see below about configuration includes.
 
 Additionally, the `~/.config/borgmatic.d/` directory works the same way as
 `/etc/borgmatic.d`.

docs/how-to/provide-your-passwords.md

@@ -25,8 +25,8 @@ storage:
 ```
 
 This uses the `MY_PASSPHRASE` environment variable as your encryption
-passphrase. Note that the `{` `}` brackets are required. Just `$MY_PASSPHRASE`
+passphrase. Note that the `{` `}` brackets are required. `$MY_PASSPHRASE` by
-will not work.
+itself will not work.
 
 In the case of `encryption_passphrase` in particular, an alternate approach
 is to use Borg's `BORG_PASSPHRASE` environment variable, which doesn't even

docs/how-to/run-arbitrary-borg-commands.md

@@ -50,7 +50,7 @@ borgmatic borg rlist --short
 ```
 
 This runs Borg's `rlist` command once on each configured borgmatic repository.
-However, the native `borgmatic rlist` action should be preferred for most use.
+(The native `borgmatic rlist` action should be preferred for most use.)
 
 What if you only want to run Borg on a single configured borgmatic repository
 when you've got several configured? Not a problem.

docs/how-to/set-up-backups.md

@@ -347,7 +347,7 @@ instead:
 sudo su -c "borgmatic --bash-completion > /usr/share/bash-completion/completions/borgmatic"
 ```
 
-Or, if you'd like to install the script for just the current user:
+Or, if you'd like to install the script for only the current user:
 
 ```bash
 mkdir --parents ~/.local/share/bash-completion/completions

docs/how-to/upgrade.md

@@ -122,9 +122,8 @@ files.
 To upgrade to a new version of Borg, you can generally install a new version
 the same way you installed the previous version, paying attention to any
 instructions included with each Borg release changelog linked from the
-[releases page](https://github.com/borgbackup/borg/releases). However, some
+[releases page](https://github.com/borgbackup/borg/releases). Some more major
-more major Borg releases require additional steps that borgmatic can help
+Borg releases require additional steps that borgmatic can help with.
-with.
 
 
 ### Borg 1.2 to 2.0

tests/unit/commands/test_borgmatic.py

@@ -397,7 +397,9 @@ def test_run_actions_does_not_raise_for_transfer_action():
 
 def test_run_actions_calls_hooks_for_prune_action():
     flexmock(module.borg_prune).should_receive('prune_archives')
-    flexmock(module.command).should_receive('execute_hook').twice()
+    flexmock(module.command).should_receive('execute_hook').times(
+        4
+    )  # Before/after extract and before/after actions.
     arguments = {
         'global': flexmock(monitoring_verbosity=1, dry_run=False),
         'prune': flexmock(stats=flexmock(), list_archives=flexmock()),
@@ -423,7 +425,9 @@ def test_run_actions_calls_hooks_for_prune_action():
 def test_run_actions_calls_hooks_for_compact_action():
     flexmock(module.borg_feature).should_receive('available').and_return(True)
     flexmock(module.borg_compact).should_receive('compact_segments')
-    flexmock(module.command).should_receive('execute_hook').twice()
+    flexmock(module.command).should_receive('execute_hook').times(
+        4
+    )  # Before/after extract and before/after actions.
     arguments = {
         'global': flexmock(monitoring_verbosity=1, dry_run=False),
         'compact': flexmock(progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock()),
@@ -448,7 +452,9 @@ def test_run_actions_calls_hooks_for_compact_action():
 
 def test_run_actions_executes_and_calls_hooks_for_create_action():
     flexmock(module.borg_create).should_receive('create_archive')
-    flexmock(module.command).should_receive('execute_hook').twice()
+    flexmock(module.command).should_receive('execute_hook').times(
+        4
+    )  # Before/after extract and before/after actions.
     flexmock(module.dispatch).should_receive('call_hooks').and_return({}).times(3)
     arguments = {
         'global': flexmock(monitoring_verbosity=1, dry_run=False),
@@ -477,7 +483,9 @@ def test_run_actions_executes_and_calls_hooks_for_create_action():
 def test_run_actions_calls_hooks_for_check_action():
     flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True)
     flexmock(module.borg_check).should_receive('check_archives')
-    flexmock(module.command).should_receive('execute_hook').twice()
+    flexmock(module.command).should_receive('execute_hook').times(
+        4
+    )  # Before/after extract and before/after actions.
     arguments = {
         'global': flexmock(monitoring_verbosity=1, dry_run=False),
         'check': flexmock(
@@ -505,7 +513,9 @@ def test_run_actions_calls_hooks_for_check_action():
 def test_run_actions_calls_hooks_for_extract_action():
     flexmock(module.validate).should_receive('repositories_match').and_return(True)
     flexmock(module.borg_extract).should_receive('extract_archive')
-    flexmock(module.command).should_receive('execute_hook').twice()
+    flexmock(module.command).should_receive('execute_hook').times(
+        4
+    )  # Before/after extract and before/after actions.
     arguments = {
         'global': flexmock(monitoring_verbosity=1, dry_run=False),
         'extract': flexmock(