Configuration files includes and merging (#148).
This commit is contained in:
parent
3cb52423d2
commit
6ff1867312
7 changed files with 253 additions and 5 deletions
4
NEWS
4
NEWS
|
@ -1,3 +1,7 @@
|
|||
1.3.0
|
||||
* #148: Configuration file includes and merging via "!include" tag to support reuse of common
|
||||
options across configuration files.
|
||||
|
||||
1.2.18
|
||||
* #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in
|
||||
borgmatic's location section.
|
||||
|
|
62
borgmatic/config/load.py
Normal file
62
borgmatic/config/load.py
Normal file
|
@ -0,0 +1,62 @@
|
|||
import os
|
||||
import logging
|
||||
|
||||
import ruamel.yaml
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_configuration(filename):
|
||||
'''
|
||||
Load the given configuration file and return its contents as a data structure of nested dicts
|
||||
and lists.
|
||||
|
||||
Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
|
||||
if there are too many recursive includes.
|
||||
'''
|
||||
yaml = ruamel.yaml.YAML(typ='safe')
|
||||
yaml.Constructor = Include_constructor
|
||||
|
||||
return yaml.load(open(filename))
|
||||
|
||||
|
||||
def include_configuration(loader, filename_node):
|
||||
'''
|
||||
Load the given YAML filename (ignoring the given loader so we can use our own), and return its
|
||||
contents as a data structure of nested dicts and lists.
|
||||
'''
|
||||
return load_configuration(os.path.expanduser(filename_node.value))
|
||||
|
||||
|
||||
class Include_constructor(ruamel.yaml.SafeConstructor):
|
||||
'''
|
||||
A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
|
||||
separate YAML configuration files. Example syntax: `retention: !include common.yaml`
|
||||
'''
|
||||
|
||||
def __init__(self, preserve_quotes=None, loader=None):
|
||||
super(Include_constructor, self).__init__(preserve_quotes, loader)
|
||||
self.add_constructor('!include', include_configuration)
|
||||
|
||||
def flatten_mapping(self, node):
|
||||
'''
|
||||
Support the special case of shallow merging included configuration into an existing mapping
|
||||
using the YAML '<<' merge key. Example syntax:
|
||||
|
||||
```
|
||||
retention:
|
||||
keep_daily: 1
|
||||
<<: !include common.yaml
|
||||
```
|
||||
'''
|
||||
representer = ruamel.yaml.representer.SafeRepresenter()
|
||||
|
||||
for index, (key_node, value_node) in enumerate(node.value):
|
||||
if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
|
||||
included_value = representer.represent_mapping(
|
||||
tag='tag:yaml.org,2002:map', mapping=self.construct_object(value_node)
|
||||
)
|
||||
node.value[index] = (key_node, included_value)
|
||||
|
||||
super(Include_constructor, self).flatten_mapping(node)
|
|
@ -3,7 +3,9 @@ import logging
|
|||
import pkg_resources
|
||||
import pykwalify.core
|
||||
import pykwalify.errors
|
||||
from ruamel import yaml
|
||||
import ruamel.yaml
|
||||
|
||||
from borgmatic.config import load
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
@ -87,9 +89,9 @@ def parse_configuration(config_filename, schema_filename):
|
|||
logging.getLogger('pykwalify').setLevel(logging.ERROR)
|
||||
|
||||
try:
|
||||
config = yaml.safe_load(open(config_filename))
|
||||
schema = yaml.safe_load(open(schema_filename))
|
||||
except yaml.error.YAMLError as error:
|
||||
config = load.load_configuration(config_filename)
|
||||
schema = load.load_configuration(schema_filename)
|
||||
except (ruamel.yaml.error.YAMLError, RecursionError) as error:
|
||||
raise Validation_error(config_filename, (str(error),))
|
||||
|
||||
# pykwalify gets angry if the example field is not a string. So rather than bend to its will,
|
||||
|
|
|
@ -27,6 +27,85 @@ configuration paths on the command-line with borgmatic's `--config` option.
|
|||
See `borgmatic --help` for more information.
|
||||
|
||||
|
||||
## Configuration includes
|
||||
|
||||
Once you have multiple different configuration files, you might want to share
|
||||
common configuration options across these files with having to copy and paste
|
||||
them. To achieve this, you can put fragments of common configuration options
|
||||
into a file, and then include or inline that file into one or more borgmatic
|
||||
configuration files.
|
||||
|
||||
Let's say that you want to include common retention configuration across all
|
||||
of your configuration files. You could do that in each configuration file with
|
||||
the following:
|
||||
|
||||
```yaml
|
||||
location:
|
||||
...
|
||||
|
||||
retention:
|
||||
!include /etc/borgmatic/common_retention.yaml
|
||||
```
|
||||
|
||||
And then the contents of `common_retention.yaml` could be:
|
||||
|
||||
```yaml
|
||||
keep_hourly: 24
|
||||
keep_daily: 7
|
||||
```
|
||||
|
||||
To prevent borgmatic from trying to load these configuration fragments by
|
||||
themselves and complaining that they are not valid configuration files, you
|
||||
should put them in a directory other than `/etc/borgmatic.d/`. (A subdirectory
|
||||
is fine.)
|
||||
|
||||
Note that this form of include must be a YAML value rather than a key. For
|
||||
example, this will not work:
|
||||
|
||||
```yaml
|
||||
location:
|
||||
...
|
||||
|
||||
# Don't do this. It won't work!
|
||||
!include /etc/borgmatic/common_retention.yaml
|
||||
```
|
||||
|
||||
But if you do want to merge in a YAML key and its values, keep reading!
|
||||
|
||||
|
||||
## Include merging
|
||||
|
||||
If you need to get even fancier and pull in common configuration options while
|
||||
potentially overriding individual options, you can perform a YAML merge of
|
||||
included configuration using the YAML `<<` key. For instance, here's an
|
||||
example of a main configuration file that pulls in two retention options via
|
||||
an include, and then overrides one of them locally:
|
||||
|
||||
```yaml
|
||||
location:
|
||||
...
|
||||
|
||||
retention:
|
||||
keep_daily: 5
|
||||
<<: !include /etc/borgmatic/common_retention.yaml
|
||||
```
|
||||
|
||||
This is what `common_retention.yaml` might look like:
|
||||
|
||||
```yaml
|
||||
keep_hourly: 24
|
||||
keep_daily: 7
|
||||
```
|
||||
|
||||
Once this include gets merged in, the resulting configuration would have a
|
||||
`keep_hourly` value of `24` and an overridden `keep_daily` value of `5`.
|
||||
|
||||
When there is a collision of an option between the local file and the merged
|
||||
include, the local file's option takes precedent. And note that this is a
|
||||
shallow merge rather than a deep merge, so the merging does not descend into
|
||||
nested values.
|
||||
|
||||
|
||||
## Related documentation
|
||||
|
||||
* [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)
|
||||
|
|
2
setup.py
2
setup.py
|
@ -1,7 +1,7 @@
|
|||
from setuptools import setup, find_packages
|
||||
|
||||
|
||||
VERSION = '1.2.18'
|
||||
VERSION = '1.3.0'
|
||||
|
||||
|
||||
setup(
|
||||
|
|
40
tests/integration/config/test_load.py
Normal file
40
tests/integration/config/test_load.py
Normal file
|
@ -0,0 +1,40 @@
|
|||
import sys
|
||||
|
||||
from flexmock import flexmock
|
||||
|
||||
from borgmatic.config import load as module
|
||||
|
||||
|
||||
def test_load_configuration_parses_contents():
|
||||
builtins = flexmock(sys.modules['builtins'])
|
||||
builtins.should_receive('open').with_args('config.yaml').and_return('key: value')
|
||||
|
||||
assert module.load_configuration('config.yaml') == {'key': 'value'}
|
||||
|
||||
|
||||
def test_load_configuration_inlines_include():
|
||||
builtins = flexmock(sys.modules['builtins'])
|
||||
builtins.should_receive('open').with_args('include.yaml').and_return('value')
|
||||
builtins.should_receive('open').with_args('config.yaml').and_return(
|
||||
'key: !include include.yaml'
|
||||
)
|
||||
|
||||
assert module.load_configuration('config.yaml') == {'key': 'value'}
|
||||
|
||||
|
||||
def test_load_configuration_merges_include():
|
||||
builtins = flexmock(sys.modules['builtins'])
|
||||
builtins.should_receive('open').with_args('include.yaml').and_return(
|
||||
'''
|
||||
foo: bar
|
||||
baz: quux
|
||||
'''
|
||||
)
|
||||
builtins.should_receive('open').with_args('config.yaml').and_return(
|
||||
'''
|
||||
foo: override
|
||||
<<: !include include.yaml
|
||||
'''
|
||||
)
|
||||
|
||||
assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}
|
|
@ -118,6 +118,67 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
|
|||
module.parse_configuration('config.yaml', 'schema.yaml')
|
||||
|
||||
|
||||
def test_parse_configuration_inlines_include():
|
||||
mock_config_and_schema(
|
||||
'''
|
||||
location:
|
||||
source_directories:
|
||||
- /home
|
||||
|
||||
repositories:
|
||||
- hostname.borg
|
||||
|
||||
retention:
|
||||
!include include.yaml
|
||||
'''
|
||||
)
|
||||
builtins = flexmock(sys.modules['builtins'])
|
||||
builtins.should_receive('open').with_args('include.yaml').and_return(
|
||||
'''
|
||||
keep_daily: 7
|
||||
keep_hourly: 24
|
||||
'''
|
||||
)
|
||||
|
||||
result = module.parse_configuration('config.yaml', 'schema.yaml')
|
||||
|
||||
assert result == {
|
||||
'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
|
||||
'retention': {'keep_daily': 7, 'keep_hourly': 24},
|
||||
}
|
||||
|
||||
|
||||
def test_parse_configuration_merges_include():
|
||||
mock_config_and_schema(
|
||||
'''
|
||||
location:
|
||||
source_directories:
|
||||
- /home
|
||||
|
||||
repositories:
|
||||
- hostname.borg
|
||||
|
||||
retention:
|
||||
keep_daily: 1
|
||||
<<: !include include.yaml
|
||||
'''
|
||||
)
|
||||
builtins = flexmock(sys.modules['builtins'])
|
||||
builtins.should_receive('open').with_args('include.yaml').and_return(
|
||||
'''
|
||||
keep_daily: 7
|
||||
keep_hourly: 24
|
||||
'''
|
||||
)
|
||||
|
||||
result = module.parse_configuration('config.yaml', 'schema.yaml')
|
||||
|
||||
assert result == {
|
||||
'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
|
||||
'retention': {'keep_daily': 1, 'keep_hourly': 24},
|
||||
}
|
||||
|
||||
|
||||
def test_parse_configuration_raises_for_missing_config_file():
|
||||
with pytest.raises(FileNotFoundError):
|
||||
module.parse_configuration('config.yaml', 'schema.yaml')
|
||||
|
|
Loading…
Reference in a new issue