Allow environment variable resolution in configuration file
- all string fields containing an environment variable like ${FOO} will be resolved - supported format ${FOO}, ${FOO:-bar} and ${FOO-bar} to allow default values if variable is not present in environment - add --no-env argument for CLI to disable the feature which is enabled by default Resolves: #546
This commit is contained in:
parent
f2c2f3139e
commit
97b5cd089d
5 changed files with 119 additions and 4 deletions
|
@ -188,6 +188,12 @@ def make_parsers():
|
|||
action='extend',
|
||||
help='One or more configuration file options to override with specified values',
|
||||
)
|
||||
global_group.add_argument(
|
||||
'--no-env',
|
||||
dest='resolve_env',
|
||||
action='store_false',
|
||||
help='Do not resolve environment variables in configuration file',
|
||||
)
|
||||
global_group.add_argument(
|
||||
'--bash-completion',
|
||||
default=False,
|
||||
|
|
|
@ -650,7 +650,7 @@ def run_actions(
|
|||
)
|
||||
|
||||
|
||||
def load_configurations(config_filenames, overrides=None):
|
||||
def load_configurations(config_filenames, overrides=None, resolve_env=True):
|
||||
'''
|
||||
Given a sequence of configuration filenames, load and validate each configuration file. Return
|
||||
the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
|
||||
|
@ -664,7 +664,7 @@ def load_configurations(config_filenames, overrides=None):
|
|||
for config_filename in config_filenames:
|
||||
try:
|
||||
configs[config_filename] = validate.parse_configuration(
|
||||
config_filename, validate.schema_filename(), overrides
|
||||
config_filename, validate.schema_filename(), overrides, resolve_env
|
||||
)
|
||||
except PermissionError:
|
||||
logs.extend(
|
||||
|
@ -892,7 +892,9 @@ def main(): # pragma: no cover
|
|||
sys.exit(0)
|
||||
|
||||
config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
|
||||
configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides)
|
||||
configs, parse_logs = load_configurations(
|
||||
config_filenames, global_arguments.overrides, global_arguments.resolve_env
|
||||
)
|
||||
|
||||
any_json_flags = any(
|
||||
getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()
|
||||
|
|
|
@ -1,7 +1,11 @@
|
|||
import io
|
||||
import os
|
||||
import re
|
||||
|
||||
import ruamel.yaml
|
||||
|
||||
_VARIABLE_PATTERN = re.compile(r'(?<!\\)\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\}')
|
||||
|
||||
|
||||
def set_values(config, keys, value):
|
||||
'''
|
||||
|
@ -77,3 +81,35 @@ def apply_overrides(config, raw_overrides):
|
|||
|
||||
for (keys, value) in overrides:
|
||||
set_values(config, keys, value)
|
||||
|
||||
|
||||
def _resolve_string(matcher):
|
||||
'''
|
||||
Get the value from environment given a matcher containing a name and an optional default value.
|
||||
If the variable is not defined in environment and no default value is provided, an Error is raised.
|
||||
'''
|
||||
name, default = matcher.group("name"), matcher.group("default")
|
||||
out = os.getenv(name, default=default)
|
||||
if out is None:
|
||||
raise ValueError("Cannot find variable ${name} in envivonment".format(name=name))
|
||||
return out
|
||||
|
||||
|
||||
def resolve_env_variables(item):
|
||||
'''
|
||||
Resolves variables like or ${FOO} from given configuration with values from process environment
|
||||
Supported formats:
|
||||
- ${FOO} will return FOO env variable
|
||||
- ${FOO-bar} or ${FOO:-bar} will return FOO env variable if it exists, else "bar"
|
||||
|
||||
If any variable is missing in environment and no default value is provided, an Error is raised.
|
||||
'''
|
||||
if isinstance(item, str):
|
||||
return _VARIABLE_PATTERN.sub(_resolve_string, item)
|
||||
if isinstance(item, list):
|
||||
for i, subitem in enumerate(item):
|
||||
item[i] = resolve_env_variables(subitem)
|
||||
if isinstance(item, dict):
|
||||
for key, value in item.items():
|
||||
item[key] = resolve_env_variables(value)
|
||||
return item
|
||||
|
|
|
@ -79,7 +79,7 @@ def apply_logical_validation(config_filename, parsed_configuration):
|
|||
)
|
||||
|
||||
|
||||
def parse_configuration(config_filename, schema_filename, overrides=None):
|
||||
def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True):
|
||||
'''
|
||||
Given the path to a config filename in YAML format, the path to a schema filename in a YAML
|
||||
rendition of JSON Schema format, a sequence of configuration file override strings in the form
|
||||
|
@ -99,6 +99,8 @@ def parse_configuration(config_filename, schema_filename, overrides=None):
|
|||
raise Validation_error(config_filename, (str(error),))
|
||||
|
||||
override.apply_overrides(config, overrides)
|
||||
if resolve_env:
|
||||
override.resolve_env_variables(config)
|
||||
normalize.normalize(config)
|
||||
|
||||
try:
|
||||
|
|
69
tests/unit/config/test_env_variables.py
Normal file
69
tests/unit/config/test_env_variables.py
Normal file
|
@ -0,0 +1,69 @@
|
|||
import pytest
|
||||
|
||||
from borgmatic.config import override as module
|
||||
|
||||
|
||||
def test_env(monkeypatch):
|
||||
monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
|
||||
config = {'key': 'Hello $MY_CUSTOM_VALUE'}
|
||||
module.resolve_env_variables(config)
|
||||
assert config == {'key': 'Hello $MY_CUSTOM_VALUE'}
|
||||
|
||||
|
||||
def test_env_braces(monkeypatch):
|
||||
monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
|
||||
config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
|
||||
module.resolve_env_variables(config)
|
||||
assert config == {'key': 'Hello foo'}
|
||||
|
||||
|
||||
def test_env_default_value(monkeypatch):
|
||||
monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False)
|
||||
config = {'key': 'Hello ${MY_CUSTOM_VALUE:-bar}'}
|
||||
module.resolve_env_variables(config)
|
||||
assert config == {'key': 'Hello bar'}
|
||||
|
||||
|
||||
def test_env_unknown(monkeypatch):
|
||||
monkeypatch.delenv("MY_CUSTOM_VALUE", raising=False)
|
||||
config = {'key': 'Hello ${MY_CUSTOM_VALUE}'}
|
||||
with pytest.raises(ValueError):
|
||||
module.resolve_env_variables(config)
|
||||
|
||||
|
||||
def test_env_full(monkeypatch):
|
||||
monkeypatch.setenv("MY_CUSTOM_VALUE", "foo")
|
||||
monkeypatch.delenv("MY_CUSTOM_VALUE2", raising=False)
|
||||
config = {
|
||||
'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
|
||||
'dict': {
|
||||
'key': 'value',
|
||||
'anotherdict': {
|
||||
'key': 'My ${MY_CUSTOM_VALUE} here',
|
||||
'other': '${MY_CUSTOM_VALUE}',
|
||||
'list': [
|
||||
'/home/${MY_CUSTOM_VALUE}/.local',
|
||||
'/var/log/',
|
||||
'/home/${MY_CUSTOM_VALUE2:-bar}/.config',
|
||||
],
|
||||
},
|
||||
},
|
||||
'list': [
|
||||
'/home/${MY_CUSTOM_VALUE}/.local',
|
||||
'/var/log/',
|
||||
'/home/${MY_CUSTOM_VALUE2-bar}/.config',
|
||||
],
|
||||
}
|
||||
module.resolve_env_variables(config)
|
||||
assert config == {
|
||||
'key': 'Hello $MY_CUSTOM_VALUE is not resolved',
|
||||
'dict': {
|
||||
'key': 'value',
|
||||
'anotherdict': {
|
||||
'key': 'My foo here',
|
||||
'other': 'foo',
|
||||
'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',],
|
||||
},
|
||||
},
|
||||
'list': ['/home/foo/.local', '/var/log/', '/home/bar/.config',],
|
||||
}
|
Loading…
Reference in a new issue