Override particular configuration options from the command-line via "--override" flag (#268).
This commit is contained in:
parent
afaabd14a8
commit
f787dfe809
10 changed files with 278 additions and 9 deletions
5
NEWS
5
NEWS
|
@ -1,3 +1,8 @@
|
|||
1.4.21.dev0
|
||||
* #268: Override particular configuration options from the command-line via "--override" flag. See
|
||||
the documentation for more information:
|
||||
https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides
|
||||
|
||||
1.4.20
|
||||
* Fix repository probing during "borgmatic init" to respect verbosity flag and remote_path option.
|
||||
* #249: Update Healthchecks/Cronitor/Cronhub monitoring integrations to fire for "check" and
|
||||
|
|
|
@ -164,6 +164,13 @@ def parse_arguments(*unparsed_arguments):
|
|||
default=None,
|
||||
help='Write log messages to this file instead of syslog',
|
||||
)
|
||||
global_group.add_argument(
|
||||
'--override',
|
||||
metavar='SECTION.OPTION=VALUE',
|
||||
nargs='+',
|
||||
dest='overrides',
|
||||
help='One or more configuration file options to override with specified values',
|
||||
)
|
||||
global_group.add_argument(
|
||||
'--version',
|
||||
dest='version',
|
||||
|
|
|
@ -372,7 +372,7 @@ def run_actions(
|
|||
yield json.loads(json_output)
|
||||
|
||||
|
||||
def load_configurations(config_filenames):
|
||||
def load_configurations(config_filenames, overrides=None):
|
||||
'''
|
||||
Given a sequence of configuration filenames, load and validate each configuration file. Return
|
||||
the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
|
||||
|
@ -386,7 +386,7 @@ def load_configurations(config_filenames):
|
|||
for config_filename in config_filenames:
|
||||
try:
|
||||
configs[config_filename] = validate.parse_configuration(
|
||||
config_filename, validate.schema_filename()
|
||||
config_filename, validate.schema_filename(), overrides
|
||||
)
|
||||
except (ValueError, OSError, validate.Validation_error) as error:
|
||||
logs.extend(
|
||||
|
@ -584,7 +584,7 @@ def main(): # pragma: no cover
|
|||
sys.exit(0)
|
||||
|
||||
config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
|
||||
configs, parse_logs = load_configurations(config_filenames)
|
||||
configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides)
|
||||
|
||||
colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
|
||||
try:
|
||||
|
|
71
borgmatic/config/override.py
Normal file
71
borgmatic/config/override.py
Normal file
|
@ -0,0 +1,71 @@
|
|||
import io
|
||||
|
||||
import ruamel.yaml
|
||||
|
||||
|
||||
def set_values(config, keys, value):
|
||||
'''
|
||||
Given a hierarchy of configuration dicts, a sequence of parsed key strings, and a string value,
|
||||
descend into the hierarchy based on the keys to set the value into the right place.
|
||||
'''
|
||||
if not keys:
|
||||
return
|
||||
|
||||
first_key = keys[0]
|
||||
if len(keys) == 1:
|
||||
config[first_key] = value
|
||||
return
|
||||
|
||||
if first_key not in config:
|
||||
config[first_key] = {}
|
||||
|
||||
set_values(config[first_key], keys[1:], value)
|
||||
|
||||
|
||||
def convert_value_type(value):
|
||||
'''
|
||||
Given a string value, determine its logical type (string, boolean, integer, etc.), and return it
|
||||
converted to that type.
|
||||
'''
|
||||
return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
|
||||
|
||||
|
||||
def parse_overrides(raw_overrides):
|
||||
'''
|
||||
Given a sequence of configuration file override strings in the form of "section.option=value",
|
||||
parse and return a sequence of tuples (keys, values), where keys is a sequence of strings. For
|
||||
instance, given the following raw overrides:
|
||||
|
||||
['section.my_option=value1', 'section.other_option=value2']
|
||||
|
||||
... return this:
|
||||
|
||||
(
|
||||
(('section', 'my_option'), 'value1'),
|
||||
(('section', 'other_option'), 'value2'),
|
||||
)
|
||||
|
||||
Raise ValueError if an override can't be parsed.
|
||||
'''
|
||||
if not raw_overrides:
|
||||
return ()
|
||||
|
||||
try:
|
||||
return tuple(
|
||||
(tuple(raw_keys.split('.')), convert_value_type(value))
|
||||
for raw_override in raw_overrides
|
||||
for raw_keys, value in (raw_override.split('=', 1),)
|
||||
)
|
||||
except ValueError:
|
||||
raise ValueError('Invalid override. Make sure you use the form: SECTION.OPTION=VALUE')
|
||||
|
||||
|
||||
def apply_overrides(config, raw_overrides):
|
||||
'''
|
||||
Given a sequence of configuration file override strings in the form of "section.option=value"
|
||||
and a configuration dict, parse each override and set it the configuration dict.
|
||||
'''
|
||||
overrides = parse_overrides(raw_overrides)
|
||||
|
||||
for (keys, value) in overrides:
|
||||
set_values(config, keys, value)
|
|
@ -6,7 +6,7 @@ import pykwalify.core
|
|||
import pykwalify.errors
|
||||
import ruamel.yaml
|
||||
|
||||
from borgmatic.config import load
|
||||
from borgmatic.config import load, override
|
||||
|
||||
|
||||
def schema_filename():
|
||||
|
@ -82,11 +82,12 @@ def remove_examples(schema):
|
|||
return schema
|
||||
|
||||
|
||||
def parse_configuration(config_filename, schema_filename):
|
||||
def parse_configuration(config_filename, schema_filename, overrides=None):
|
||||
'''
|
||||
Given the path to a config filename in YAML format and the path to a schema filename in
|
||||
pykwalify YAML schema format, return the parsed configuration as a data structure of nested
|
||||
dicts and lists corresponding to the schema. Example return value:
|
||||
Given the path to a config filename in YAML format, the path to a schema filename in pykwalify
|
||||
YAML schema format, a sequence of configuration file override strings in the form of
|
||||
"section.option=value", return the parsed configuration as a data structure of nested dicts and
|
||||
lists corresponding to the schema. Example return value:
|
||||
|
||||
{'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
|
||||
'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
|
||||
|
@ -102,6 +103,8 @@ def parse_configuration(config_filename, schema_filename):
|
|||
except (ruamel.yaml.error.YAMLError, RecursionError) as error:
|
||||
raise Validation_error(config_filename, (str(error),))
|
||||
|
||||
override.apply_overrides(config, overrides)
|
||||
|
||||
validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
|
||||
parsed_result = validator.validate(raise_exception=False)
|
||||
|
||||
|
|
|
@ -115,6 +115,40 @@ Note that this `<<` include merging syntax is only for merging in mappings
|
|||
directly, please see the section above about standard includes.
|
||||
|
||||
|
||||
## Configuration overrides
|
||||
|
||||
In more complex multi-application setups, you may want to override particular
|
||||
borgmatic configuration file options at the time you run borgmatic. For
|
||||
instance, you could reuse a common configuration file for multiple
|
||||
applications, but then set the repository for each application at runtime. Or
|
||||
you might want to try a variant of an option for testing purposes without
|
||||
actually touching your configuration file.
|
||||
|
||||
Whatever the reason, you can override borgmatic configuration options at the
|
||||
command-line via the `--override` flag. Here's an example:
|
||||
|
||||
```bash
|
||||
borgmatic create --override location.remote_path=borg1
|
||||
```
|
||||
|
||||
What this does is load your configuration files, and for each one, disregard
|
||||
the configured value for the `remote_path` option in the `location` section,
|
||||
and use the value of `borg1` instead.
|
||||
|
||||
Note that the value is parsed as an actual YAML string, so you can even set
|
||||
list values by using brackets. For instance:
|
||||
|
||||
```bash
|
||||
borgmatic create --override location.repositories=[test1.borg,test2.borg]
|
||||
```
|
||||
|
||||
There is not currently a way to override a single element of a list without
|
||||
replacing the whole list.
|
||||
|
||||
Be sure to quote your overrides if they contain spaces or other characters
|
||||
that your shell may interpret.
|
||||
|
||||
|
||||
## Related documentation
|
||||
|
||||
* [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
|
||||
|
|
2
setup.py
2
setup.py
|
@ -1,6 +1,6 @@
|
|||
from setuptools import find_packages, setup
|
||||
|
||||
VERSION = '1.4.20'
|
||||
VERSION = '1.4.21.dev0'
|
||||
|
||||
|
||||
setup(
|
||||
|
|
40
tests/integration/config/test_override.py
Normal file
40
tests/integration/config/test_override.py
Normal file
|
@ -0,0 +1,40 @@
|
|||
import pytest
|
||||
|
||||
from borgmatic.config import override as module
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'value,expected_result',
|
||||
(
|
||||
('thing', 'thing'),
|
||||
('33', 33),
|
||||
('33b', '33b'),
|
||||
('true', True),
|
||||
('false', False),
|
||||
('[foo]', ['foo']),
|
||||
('[foo, bar]', ['foo', 'bar']),
|
||||
),
|
||||
)
|
||||
def test_convert_value_type_coerces_values(value, expected_result):
|
||||
assert module.convert_value_type(value) == expected_result
|
||||
|
||||
|
||||
def test_apply_overrides_updates_config():
|
||||
raw_overrides = [
|
||||
'section.key=value1',
|
||||
'other_section.thing=value2',
|
||||
'section.nested.key=value3',
|
||||
'new.foo=bar',
|
||||
]
|
||||
config = {
|
||||
'section': {'key': 'value', 'other': 'other_value'},
|
||||
'other_section': {'thing': 'thing_value'},
|
||||
}
|
||||
|
||||
module.apply_overrides(config, raw_overrides)
|
||||
|
||||
assert config == {
|
||||
'section': {'key': 'value1', 'other': 'other_value', 'nested': {'key': 'value3'}},
|
||||
'other_section': {'thing': 'value2'},
|
||||
'new': {'foo': 'bar'},
|
||||
}
|
|
@ -212,3 +212,30 @@ def test_parse_configuration_raises_for_validation_error():
|
|||
|
||||
with pytest.raises(module.Validation_error):
|
||||
module.parse_configuration('config.yaml', 'schema.yaml')
|
||||
|
||||
|
||||
def test_parse_configuration_applies_overrides():
|
||||
mock_config_and_schema(
|
||||
'''
|
||||
location:
|
||||
source_directories:
|
||||
- /home
|
||||
|
||||
repositories:
|
||||
- hostname.borg
|
||||
|
||||
local_path: borg1
|
||||
'''
|
||||
)
|
||||
|
||||
result = module.parse_configuration(
|
||||
'config.yaml', 'schema.yaml', overrides=['location.local_path=borg2']
|
||||
)
|
||||
|
||||
assert result == {
|
||||
'location': {
|
||||
'source_directories': ['/home'],
|
||||
'repositories': ['hostname.borg'],
|
||||
'local_path': 'borg2',
|
||||
}
|
||||
}
|
||||
|
|
82
tests/unit/config/test_override.py
Normal file
82
tests/unit/config/test_override.py
Normal file
|
@ -0,0 +1,82 @@
|
|||
import pytest
|
||||
from flexmock import flexmock
|
||||
|
||||
from borgmatic.config import override as module
|
||||
|
||||
|
||||
def test_set_values_with_empty_keys_bails():
|
||||
config = {}
|
||||
|
||||
module.set_values(config, keys=(), value='value')
|
||||
|
||||
assert config == {}
|
||||
|
||||
|
||||
def test_set_values_with_one_key_sets_it_into_config():
|
||||
config = {}
|
||||
|
||||
module.set_values(config, keys=('key',), value='value')
|
||||
|
||||
assert config == {'key': 'value'}
|
||||
|
||||
|
||||
def test_set_values_with_one_key_overwrites_existing_key():
|
||||
config = {'key': 'old_value', 'other': 'other_value'}
|
||||
|
||||
module.set_values(config, keys=('key',), value='value')
|
||||
|
||||
assert config == {'key': 'value', 'other': 'other_value'}
|
||||
|
||||
|
||||
def test_set_values_with_multiple_keys_creates_hierarchy():
|
||||
config = {}
|
||||
|
||||
module.set_values(config, ('section', 'key'), 'value')
|
||||
|
||||
assert config == {'section': {'key': 'value'}}
|
||||
|
||||
|
||||
def test_set_values_with_multiple_keys_updates_hierarchy():
|
||||
config = {'section': {'other': 'other_value'}}
|
||||
module.set_values(config, ('section', 'key'), 'value')
|
||||
|
||||
assert config == {'section': {'key': 'value', 'other': 'other_value'}}
|
||||
|
||||
|
||||
def test_parse_overrides_splits_keys_and_values():
|
||||
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
|
||||
raw_overrides = ['section.my_option=value1', 'section.other_option=value2']
|
||||
expected_result = (
|
||||
(('section', 'my_option'), 'value1'),
|
||||
(('section', 'other_option'), 'value2'),
|
||||
)
|
||||
|
||||
module.parse_overrides(raw_overrides) == expected_result
|
||||
|
||||
|
||||
def test_parse_overrides_allows_value_with_equal_sign():
|
||||
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
|
||||
raw_overrides = ['section.option=this===value']
|
||||
expected_result = ((('section', 'option'), 'this===value'),)
|
||||
|
||||
module.parse_overrides(raw_overrides) == expected_result
|
||||
|
||||
|
||||
def test_parse_overrides_raises_on_missing_equal_sign():
|
||||
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
|
||||
raw_overrides = ['section.option']
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
module.parse_overrides(raw_overrides)
|
||||
|
||||
|
||||
def test_parse_overrides_allows_value_with_single_key():
|
||||
flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
|
||||
raw_overrides = ['option=value']
|
||||
expected_result = ((('option',), 'value'),)
|
||||
|
||||
module.parse_overrides(raw_overrides) == expected_result
|
||||
|
||||
|
||||
def test_parse_overrides_handles_empty_overrides():
|
||||
module.parse_overrides(raw_overrides=None) == ()
|
Loading…
Reference in a new issue