Allow environment variable resolution in configuration file

- all string fields containing an environment variable like ${FOO} will
  be resolved
- supported format ${FOO}, ${FOO:-bar} and ${FOO-bar} to allow default
  values if variable is not present in environment
- add --no-env argument for CLI to disable the feature which is enabled
  by default

Resolves: #546

borgmatic/commands/arguments.py

@@ -188,6 +188,12 @@ def make_parsers():
         action='extend',
         help='One or more configuration file options to override with specified values',
     )
+    global_group.add_argument(
+        '--no-env',
+        dest='resolve_env',
+        action='store_false',
+        help='Do not resolve environment variables in configuration file',
+    )
     global_group.add_argument(
         '--bash-completion',
         default=False,

borgmatic/commands/borgmatic.py

@@ -650,7 +650,7 @@ def run_actions(
             )
 
 
-def load_configurations(config_filenames, overrides=None):
+def load_configurations(config_filenames, overrides=None, resolve_env=True):
     '''
     Given a sequence of configuration filenames, load and validate each configuration file. Return
     the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
@@ -664,7 +664,7 @@ def load_configurations(config_filenames, overrides=None):
     for config_filename in config_filenames:
         try:
             configs[config_filename] = validate.parse_configuration(
-                config_filename, validate.schema_filename(), overrides
+                config_filename, validate.schema_filename(), overrides, resolve_env
             )
         except PermissionError:
             logs.extend(
@@ -892,7 +892,9 @@ def main():  # pragma: no cover
         sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
-    configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides)
+    configs, parse_logs = load_configurations(
+        config_filenames, global_arguments.overrides, global_arguments.resolve_env
+    )
 
     any_json_flags = any(
         getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()

borgmatic/config/override.py

@@ -1,7 +1,11 @@
 import io
+import os
+import re
 
 import ruamel.yaml
 
+_VARIABLE_PATTERN = re.compile(r'(?<!\\)\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\}')
+
 
 def set_values(config, keys, value):
     '''
@@ -77,3 +81,35 @@ def apply_overrides(config, raw_overrides):
 
     for (keys, value) in overrides:
         set_values(config, keys, value)
+
+
+def _resolve_string(matcher):
+    '''
+    Get the value from environment given a matcher containing a name and an optional default value.
+    If the variable is not defined in environment and no default value is provided, an Error is raised.
+    '''
+    name, default = matcher.group("name"), matcher.group("default")
+    out = os.getenv(name, default=default)
+    if out is None:
+        raise ValueError("Cannot find variable ${name} in envivonment".format(name=name))
+    return out
+
+
+def resolve_env_variables(item):
+    '''
+    Resolves variables like or ${FOO} from given configuration with values from process environment
+    Supported formats:
+     - ${FOO} will return FOO env variable
+     - ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
+
+    If any variable is missing in environment and no default value is provided, an Error is raised.
+    '''
+    if isinstance(item, str):
+        return _VARIABLE_PATTERN.sub(_resolve_string, item)
+    if isinstance(item, list):
+        for i, subitem in enumerate(item):
+            item[i] = resolve_env_variables(subitem)
+    if isinstance(item, dict):
+        for key, value in item.items():
+            item[key] = resolve_env_variables(value)
+    return item

borgmatic/config/validate.py

@@ -79,7 +79,7 @@ def apply_logical_validation(config_filename, parsed_configuration):
             )
 
 
-def parse_configuration(config_filename, schema_filename, overrides=None):
+def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True):
     '''
     Given the path to a config filename in YAML format, the path to a schema filename in a YAML
     rendition of JSON Schema format, a sequence of configuration file override strings in the form
@@ -99,6 +99,8 @@ def parse_configuration(config_filename, schema_filename, overrides=None):
         raise Validation_error(config_filename, (str(error),))
 
     override.apply_overrides(config, overrides)
+    if resolve_env:
+        override.resolve_env_variables(config)
     normalize.normalize(config)
 
     try:

tests/unit/config/test_env_variables.py

@@ -0,0 +1,69 @@
+import pytest
+
+from borgmatic.config import override as module
+
+
+def test_env(monkeypatch):
+    monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
+    config = {'key': 'Hello $MY_CUSTOM_VALUE'}
+    module.resolve_env_variables(config)
+    assert config == {'key': 'Hello $MY_CUSTOM_VALUE'}
+
+
+def test_env_braces(monkeypatch):
+    monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
+    config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
+    module.resolve_env_variables(config)
+    assert config == {'key': 'Hello foo'}
+
+
+def test_env_default_value(monkeypatch):
+    monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False)
+    config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'}
+    module.resolve_env_variables(config)
+    assert config == {'key': 'Hello bar'}
+
+
+def test_env_unknown(monkeypatch):
+    monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False)
+    config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
+    with pytest.raises(ValueError):
+        module.resolve_env_variables(config)
+
+
+def test_env_full(monkeypatch):
+    monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
+    monkeypatch.delenv("MY_CUSTOM_VALUE2", raising=False)
+    config = {
+        'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
+        'dict': {
+            'key': 'value',
+            'anotherdict': {
+                'key': 'My ${MY_CUSTOM_VALUE} here',
+                'other': '${MY_CUSTOM_VALUE}',
+                'list': [
+                    '/home/${MY_CUSTOM_VALUE}/.local',
+                    '/var/log/',
+                    '/home/${MY_CUSTOM_VALUE2:-bar}/.config',
+                ],
+            },
+        },
+        'list': [
+            '/home/${MY_CUSTOM_VALUE}/.local',
+            '/var/log/',
+            '/home/${MY_CUSTOM_VALUE2-bar}/.config',
+        ],
+    }
+    module.resolve_env_variables(config)
+    assert config == {
+        'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
+        'dict': {
+            'key': 'value',
+            'anotherdict': {
+                'key': 'My foo here',
+                'other': 'foo',
+                'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',],
+            },
+        },
+        'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',],
+    }