Configuration files includes and merging (#148).

This commit is contained in:
Dan Helfman 2019-03-06 12:06:27 -08:00
parent 3cb52423d2
commit 6ff1867312
7 changed files with 253 additions and 5 deletions

4
NEWS
View file

@ -1,3 +1,7 @@
1.3.0
* #148: Configuration file includes and merging via "!include" tag to support reuse of common
options across configuration files.
1.2.18
* #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in
borgmatic's location section.

62
borgmatic/config/load.py Normal file
View file

@ -0,0 +1,62 @@
import os
import logging
import ruamel.yaml
logger = logging.getLogger(__name__)
def load_configuration(filename):
'''
Load the given configuration file and return its contents as a data structure of nested dicts
and lists.
Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
if there are too many recursive includes.
'''
yaml = ruamel.yaml.YAML(typ='safe')
yaml.Constructor = Include_constructor
return yaml.load(open(filename))
def include_configuration(loader, filename_node):
'''
Load the given YAML filename (ignoring the given loader so we can use our own), and return its
contents as a data structure of nested dicts and lists.
'''
return load_configuration(os.path.expanduser(filename_node.value))
class Include_constructor(ruamel.yaml.SafeConstructor):
'''
A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
separate YAML configuration files. Example syntax: `retention: !include common.yaml`
'''
def __init__(self, preserve_quotes=None, loader=None):
super(Include_constructor, self).__init__(preserve_quotes, loader)
self.add_constructor('!include', include_configuration)
def flatten_mapping(self, node):
'''
Support the special case of shallow merging included configuration into an existing mapping
using the YAML '<<' merge key. Example syntax:
```
retention:
keep_daily: 1
<<: !include common.yaml
```
'''
representer = ruamel.yaml.representer.SafeRepresenter()
for index, (key_node, value_node) in enumerate(node.value):
if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
included_value = representer.represent_mapping(
tag='tag:yaml.org,2002:map', mapping=self.construct_object(value_node)
)
node.value[index] = (key_node, included_value)
super(Include_constructor, self).flatten_mapping(node)

View file

@ -3,7 +3,9 @@ import logging
import pkg_resources
import pykwalify.core
import pykwalify.errors
from ruamel import yaml
import ruamel.yaml
from borgmatic.config import load
logger = logging.getLogger(__name__)
@ -87,9 +89,9 @@ def parse_configuration(config_filename, schema_filename):
logging.getLogger('pykwalify').setLevel(logging.ERROR)
try:
config = yaml.safe_load(open(config_filename))
schema = yaml.safe_load(open(schema_filename))
except yaml.error.YAMLError as error:
config = load.load_configuration(config_filename)
schema = load.load_configuration(schema_filename)
except (ruamel.yaml.error.YAMLError, RecursionError) as error:
raise Validation_error(config_filename, (str(error),))
# pykwalify gets angry if the example field is not a string. So rather than bend to its will,

View file

@ -27,6 +27,85 @@ configuration paths on the command-line with borgmatic's `--config` option.
See `borgmatic --help` for more information.
## Configuration includes
Once you have multiple different configuration files, you might want to share
common configuration options across these files with having to copy and paste
them. To achieve this, you can put fragments of common configuration options
into a file, and then include or inline that file into one or more borgmatic
configuration files.
Let's say that you want to include common retention configuration across all
of your configuration files. You could do that in each configuration file with
the following:
```yaml
location:
...
retention:
!include /etc/borgmatic/common_retention.yaml
```
And then the contents of `common_retention.yaml` could be:
```yaml
keep_hourly: 24
keep_daily: 7
```
To prevent borgmatic from trying to load these configuration fragments by
themselves and complaining that they are not valid configuration files, you
should put them in a directory other than `/etc/borgmatic.d/`. (A subdirectory
is fine.)
Note that this form of include must be a YAML value rather than a key. For
example, this will not work:
```yaml
location:
...
# Don't do this. It won't work!
!include /etc/borgmatic/common_retention.yaml
```
But if you do want to merge in a YAML key and its values, keep reading!
## Include merging
If you need to get even fancier and pull in common configuration options while
potentially overriding individual options, you can perform a YAML merge of
included configuration using the YAML `<<` key. For instance, here's an
example of a main configuration file that pulls in two retention options via
an include, and then overrides one of them locally:
```yaml
location:
...
retention:
keep_daily: 5
<<: !include /etc/borgmatic/common_retention.yaml
```
This is what `common_retention.yaml` might look like:
```yaml
keep_hourly: 24
keep_daily: 7
```
Once this include gets merged in, the resulting configuration would have a
`keep_hourly` value of `24` and an overridden `keep_daily` value of `5`.
When there is a collision of an option between the local file and the merged
include, the local file's option takes precedent. And note that this is a
shallow merge rather than a deep merge, so the merging does not descend into
nested values.
## Related documentation
* [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)

View file

@ -1,7 +1,7 @@
from setuptools import setup, find_packages
VERSION = '1.2.18'
VERSION = '1.3.0'
setup(

View file

@ -0,0 +1,40 @@
import sys
from flexmock import flexmock
from borgmatic.config import load as module
def test_load_configuration_parses_contents():
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('config.yaml').and_return('key: value')
assert module.load_configuration('config.yaml') == {'key': 'value'}
def test_load_configuration_inlines_include():
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('include.yaml').and_return('value')
builtins.should_receive('open').with_args('config.yaml').and_return(
'key: !include include.yaml'
)
assert module.load_configuration('config.yaml') == {'key': 'value'}
def test_load_configuration_merges_include():
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('include.yaml').and_return(
'''
foo: bar
baz: quux
'''
)
builtins.should_receive('open').with_args('config.yaml').and_return(
'''
foo: override
<<: !include include.yaml
'''
)
assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}

View file

@ -118,6 +118,67 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
module.parse_configuration('config.yaml', 'schema.yaml')
def test_parse_configuration_inlines_include():
mock_config_and_schema(
'''
location:
source_directories:
- /home
repositories:
- hostname.borg
retention:
!include include.yaml
'''
)
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('include.yaml').and_return(
'''
keep_daily: 7
keep_hourly: 24
'''
)
result = module.parse_configuration('config.yaml', 'schema.yaml')
assert result == {
'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
'retention': {'keep_daily': 7, 'keep_hourly': 24},
}
def test_parse_configuration_merges_include():
mock_config_and_schema(
'''
location:
source_directories:
- /home
repositories:
- hostname.borg
retention:
keep_daily: 1
<<: !include include.yaml
'''
)
builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('include.yaml').and_return(
'''
keep_daily: 7
keep_hourly: 24
'''
)
result = module.parse_configuration('config.yaml', 'schema.yaml')
assert result == {
'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
'retention': {'keep_daily': 1, 'keep_hourly': 24},
}
def test_parse_configuration_raises_for_missing_config_file():
with pytest.raises(FileNotFoundError):
module.parse_configuration('config.yaml', 'schema.yaml')