Basic YAML generating / validating / converting to.

This commit is contained in:
Dan Helfman 2017-07-08 22:33:51 -07:00
parent 483bd50bdf
commit f19a40ef9c
16 changed files with 327 additions and 40 deletions

5
NEWS
View file

@ -1,8 +1,9 @@
1.1.0 1.1.0.dev0
* Switched config file format to YAML. Run convert-borgmatic-config to upgrade.
* Dropped Python 2 support. Now Python 3 only.
* #18: Fix for README mention of sample files not included in package. * #18: Fix for README mention of sample files not included in package.
* #22: Sample files for triggering borgmatic from a systemd timer. * #22: Sample files for triggering borgmatic from a systemd timer.
* Dropped Python 2 support. Now Python 3 only.
* Added logo. * Added logo.
1.0.3 1.0.3

View file

View file

@ -5,7 +5,7 @@ from subprocess import CalledProcessError
import sys import sys
from borgmatic import borg from borgmatic import borg
from borgmatic.config.yaml import parse_configuration, schema_filename from borgmatic.config.validate import parse_configuration, schema_filename
DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
@ -14,9 +14,8 @@ DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
def parse_arguments(*arguments): def parse_arguments(*arguments):
''' '''
Given the name of the command with which this script was invoked and command-line arguments, Given command-line arguments with which this script was invoked, parse the arguments and return
parse the arguments and return them as an ArgumentParser instance. Use the command name to them as an ArgumentParser instance.
determine the default configuration and excludes paths.
''' '''
parser = ArgumentParser() parser = ArgumentParser()
parser.add_argument( parser.add_argument(

View file

@ -0,0 +1,54 @@
from __future__ import print_function
from argparse import ArgumentParser
import os
from subprocess import CalledProcessError
import sys
from ruamel import yaml
from borgmatic import borg
from borgmatic.config import convert, generate, legacy, validate
DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
def parse_arguments(*arguments):
'''
Given command-line arguments with which this script was invoked, parse the arguments and return
them as an ArgumentParser instance.
'''
parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.')
parser.add_argument(
'-s', '--source',
dest='source_filename',
default=DEFAULT_SOURCE_CONFIG_FILENAME,
help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
)
parser.add_argument(
'-d', '--destination',
dest='destination_filename',
default=DEFAULT_DESTINATION_CONFIG_FILENAME,
help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
)
return parser.parse_args(arguments)
def main():
try:
args = parse_arguments(*sys.argv[1:])
source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT)
schema = yaml.round_trip_load(open(validate.schema_filename()).read())
destination_config = convert.convert_legacy_parsed_config(source_config, schema)
generate.write_configuration(args.destination_filename, destination_config)
# TODO: As a backstop, check that the written config can actually be read and parsed, and
# that it matches the destination config data structure that was written.
except (ValueError, OSError) as error:
print(error, file=sys.stderr)
sys.exit(1)

View file

@ -0,0 +1,41 @@
from ruamel import yaml
from borgmatic.config import generate
def _convert_section(source_section_config, section_schema):
'''
Given a legacy Parsed_config instance for a single section, convert it to its corresponding
yaml.comments.CommentedMap representation in preparation for actual serialization to YAML.
Additionally, use the section schema as a source of helpful comments to include within the
returned CommentedMap.
'''
destination_section_config = yaml.comments.CommentedMap(source_section_config)
generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT)
return destination_section_config
def convert_legacy_parsed_config(source_config, schema):
'''
Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its
corresponding yaml.comments.CommentedMap representation in preparation for actual serialization
to YAML.
Additionally, use the given schema as a source of helpful comments to include within the
returned CommentedMap.
'''
destination_config = yaml.comments.CommentedMap([
(section_name, _convert_section(section_config, schema['map'][section_name]))
for section_name, section_config in source_config._asdict().items()
])
destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
if source_config.consistency['checks']:
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
generate.add_comments_to_configuration(destination_config, schema)
return destination_config

View file

@ -0,0 +1,90 @@
from collections import OrderedDict
from ruamel import yaml
INDENT = 4
def write_configuration(config_filename, config):
'''
Given a target config filename and a config data structure of nested OrderedDicts, write out the
config to file as YAML.
'''
with open(config_filename, 'w') as config_file:
config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT))
def _insert_newline_before_comment(config, field_name):
'''
Using some ruamel.yaml black magic, insert a blank line in the config right befor the given
field and its comments.
'''
config.ca.items[field_name][1].insert(
0,
yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None),
)
def add_comments_to_configuration(config, schema, indent=0):
'''
Using descriptions from a schema as a source, add those descriptions as comments to the given
config before each field. This function only adds comments for the top-most config map level.
Indent the comment the given number of characters.
'''
for index, field_name in enumerate(config.keys()):
field_schema = schema['map'].get(field_name, {})
description = field_schema.get('desc')
# No description to use? Skip it.
if not schema or not description:
continue
config.yaml_set_comment_before_after_key(
key=field_name,
before=description,
indent=indent,
)
if index > 0:
_insert_newline_before_comment(config, field_name)
def _section_schema_to_sample_configuration(section_schema):
'''
Given the schema for a particular config section, generate and return sample config for that
section. Include comments for each field based on the schema "desc" description.
'''
section_config = yaml.comments.CommentedMap([
(field_name, field_schema['example'])
for field_name, field_schema in section_schema['map'].items()
])
add_comments_to_configuration(section_config, section_schema, indent=INDENT)
return section_config
def _schema_to_sample_configuration(schema):
'''
Given a loaded configuration schema, generate and return sample config for it. Include comments
for each section based on the schema "desc" description.
'''
config = yaml.comments.CommentedMap([
(section_name, _section_schema_to_sample_configuration(section_schema))
for section_name, section_schema in schema['map'].items()
])
add_comments_to_configuration(config, schema)
return config
def generate_sample_configuration(config_filename, schema_filename):
'''
Given a target config filename and the path to a schema filename in pykwalify YAML schema
format, write out a sample configuration file based on that schema.
'''
schema = yaml.round_trip_load(open(schema_filename))
config = _schema_to_sample_configuration(schema)
write_configuration(config_filename, config)

View file

@ -1,48 +1,110 @@
name: Borgmatic configuration file schema
map: map:
location: location:
desc: |
Where to look for files to backup, and where to store those backups. See
https://borgbackup.readthedocs.io/en/stable/quickstart.html and
https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details.
required: True required: True
map: map:
source_directories: source_directories:
required: True required: True
seq: seq:
- type: scalar - type: scalar
desc: List of source directories to backup. Globs are expanded.
example:
- /home
- /etc
- /var/log/syslog*
one_file_system: one_file_system:
type: bool type: bool
desc: Stay in same file system (do not cross mount points).
example: yes
remote_path: remote_path:
type: scalar type: scalar
desc: Alternate Borg remote executable. Defaults to "borg".
example: borg1
repository: repository:
required: True required: True
type: scalar type: scalar
desc: Path to local or remote repository.
example: user@backupserver:sourcehostname.borg
storage: storage:
desc: |
Repository storage options. See
https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and
https://borgbackup.readthedocs.io/en/stable/usage.html#environment-variables for details.
map: map:
encryption_passphrase: encryption_passphrase:
type: scalar type: scalar
desc: |
Passphrase to unlock the encryption key with. Only use on repositories that were
initialized with passphrase/repokey encryption. Quote the value if it contains
punctuation, so it parses correctly. And backslash any quote or backslash
literals as well.
example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
compression: compression:
type: scalar type: scalar
desc: |
Type of compression to use when creating archives. See
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
Defaults to no compression.
example: lz4
umask: umask:
type: scalar type: scalar
desc: Umask to be used for borg create.
example: 0077
retention: retention:
desc: |
Retention policy for how many backups to keep in each category. See
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
map: map:
keep_within: keep_within:
type: scalar type: scalar
desc: Keep all archives within this time interval.
example: 3H
keep_hourly: keep_hourly:
type: int type: int
desc: Number of hourly archives to keep.
example: 24
keep_daily: keep_daily:
type: int type: int
desc: Number of daily archives to keep.
example: 7
keep_weekly: keep_weekly:
type: int type: int
desc: Number of weekly archives to keep.
example: 4
keep_monthly: keep_monthly:
type: int type: int
desc: Number of monthly archives to keep.
example: 6
keep_yearly: keep_yearly:
type: int type: int
desc: Number of yearly archives to keep.
example: 1
prefix: prefix:
type: scalar type: scalar
desc: When pruning, only consider archive names starting with this prefix.
example: sourcehostname
consistency: consistency:
desc: |
Consistency checks to run after backups. See
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
map: map:
checks: checks:
seq: seq:
- type: str - type: str
enum: ['repository', 'archives', 'disabled'] enum: ['repository', 'archives', 'disabled']
unique: True unique: True
desc: |
List of consistency checks to run: "repository", "archives", or both. Defaults
to both. Set to "disabled" to disable all consistency checks. See
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
example:
- repository
- archives
check_last: check_last:
type: int type: int
desc: Restrict the number of checked archives to the last n.
example: 3

View file

@ -5,7 +5,7 @@ import warnings
import pkg_resources import pkg_resources
import pykwalify.core import pykwalify.core
import pykwalify.errors import pykwalify.errors
import ruamel.yaml.error from ruamel import yaml
def schema_filename(): def schema_filename():
@ -38,20 +38,18 @@ def parse_configuration(config_filename, schema_filename):
Raise FileNotFoundError if the file does not exist, PermissionError if the user does not Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
have permissions to read the file, or Validation_error if the config does not match the schema. have permissions to read the file, or Validation_error if the config does not match the schema.
''' '''
warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
try: try:
validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename]) schema = yaml.round_trip_load(open(schema_filename))
except pykwalify.errors.CoreError as error: except yaml.error.YAMLError as error:
if 'do not exists on disk' in str(error): raise Validation_error(config_filename, (str(error),))
raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
if 'Unable to load any data' in str(error):
# If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
# So reach back to the originating exception from ruamel.yaml for something more useful.
raise Validation_error(config_filename, (error.__context__,))
raise
# pykwalify gets angry if the example field is not a string. So rather than bend to its will,
# simply remove all examples before passing the schema to pykwalify.
for section_name, section_schema in schema['map'].items():
for field_name, field_schema in section_schema['map'].items():
field_schema.pop('example')
validator = pykwalify.core.Core(source_file=config_filename, schema_data=schema)
parsed_result = validator.validate(raise_exception=False) parsed_result = validator.validate(raise_exception=False)
if validator.validation_errors: if validator.validation_errors:
@ -73,12 +71,3 @@ def display_validation_error(validation_error):
for error in validation_error.error_messages: for error in validation_error.error_messages:
print(error, file=sys.stderr) print(error, file=sys.stderr)
# FOR TESTING
if __name__ == '__main__':
try:
configuration = parse_configuration('sample/config.yaml', schema_filename())
print(configuration)
except Validation_error as error:
display_validation_error(error)

View file

@ -4,7 +4,7 @@ import sys
from flexmock import flexmock from flexmock import flexmock
import pytest import pytest
from borgmatic import command as module from borgmatic.commands import borgmatic as module
def test_parse_arguments_with_no_arguments_uses_defaults(): def test_parse_arguments_with_no_arguments_uses_defaults():

View file

@ -6,7 +6,7 @@ import os
from flexmock import flexmock from flexmock import flexmock
import pytest import pytest
from borgmatic.config import yaml as module from borgmatic.config import validate as module
def test_schema_filename_returns_plausable_path(): def test_schema_filename_returns_plausable_path():
@ -18,13 +18,13 @@ def test_schema_filename_returns_plausable_path():
def mock_config_and_schema(config_yaml): def mock_config_and_schema(config_yaml):
''' '''
Set up mocks for the config config YAML string and the default schema so that pykwalify consumes Set up mocks for the config config YAML string and the default schema so that pykwalify consumes
them when parsing the configuration. This is a little brittle in that it's relying on pykwalify them when parsing the configuration. This is a little brittle in that it's relying on the code
to open() the respective files in a particular order. under test to open() the respective files in a particular order.
''' '''
config_stream = io.StringIO(config_yaml)
schema_stream = open(module.schema_filename()) schema_stream = open(module.schema_filename())
config_stream = io.StringIO(config_yaml)
builtins = flexmock(sys.modules['builtins']).should_call('open').mock builtins = flexmock(sys.modules['builtins']).should_call('open').mock
builtins.should_receive('open').and_return(config_stream).and_return(schema_stream) builtins.should_receive('open').and_return(schema_stream).and_return(config_stream)
flexmock(os.path).should_receive('exists').and_return(True) flexmock(os.path).should_receive('exists').and_return(True)
@ -87,7 +87,8 @@ def test_parse_configuration_raises_for_missing_config_file():
def test_parse_configuration_raises_for_missing_schema_file(): def test_parse_configuration_raises_for_missing_schema_file():
mock_config_and_schema('') mock_config_and_schema('')
flexmock(os.path).should_receive('exists').with_args('schema.yaml').and_return(False) builtins = flexmock(sys.modules['builtins'])
builtins.should_receive('open').with_args('schema.yaml').and_raise(FileNotFoundError)
with pytest.raises(FileNotFoundError): with pytest.raises(FileNotFoundError):
module.parse_configuration('config.yaml', 'schema.yaml') module.parse_configuration('config.yaml', 'schema.yaml')

View file

@ -0,0 +1,44 @@
from collections import defaultdict, OrderedDict, namedtuple
from borgmatic.config import convert as module
Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency'))
def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
source_config = Parsed_config(
location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]),
storage=OrderedDict([('encryption_passphrase', 'supersecret')]),
retention=OrderedDict([('keep_daily', 7)]),
consistency=OrderedDict([('checks', 'repository')]),
)
schema = {'map': defaultdict(lambda: {'map': {}})}
destination_config = module.convert_legacy_parsed_config(source_config, schema)
assert destination_config == OrderedDict([
('location', OrderedDict([('source_directories', ['/home']), ('repository', 'hostname.borg')])),
('storage', OrderedDict([('encryption_passphrase', 'supersecret')])),
('retention', OrderedDict([('keep_daily', 7)])),
('consistency', OrderedDict([('checks', ['repository'])])),
])
def test_convert_legacy_parsed_config_splits_space_separated_values():
source_config = Parsed_config(
location=OrderedDict([('source_directories', '/home /etc')]),
storage=OrderedDict(),
retention=OrderedDict(),
consistency=OrderedDict([('checks', 'repository archives')]),
)
schema = {'map': defaultdict(lambda: {'map': {}})}
destination_config = module.convert_legacy_parsed_config(source_config, schema)
assert destination_config == OrderedDict([
('location', OrderedDict([('source_directories', ['/home', '/etc'])])),
('storage', OrderedDict()),
('retention', OrderedDict()),
('consistency', OrderedDict([('checks', ['repository', 'archives'])])),
])

View file

@ -16,8 +16,10 @@ location:
#storage: #storage:
# Passphrase to unlock the encryption key with. Only use on repositories # Passphrase to unlock the encryption key with. Only use on repositories
# that were initialized with passphrase/repokey encryption. # that were initialized with passphrase/repokey encryption. Quote the value
#encryption_passphrase: foo # if it contains punctuation so it parses correctly. And backslash any
# quote or backslash literals as well.
#encryption_passphrase: "foo"
# Type of compression to use when creating archives. See # Type of compression to use when creating archives. See
# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create

View file

@ -1,7 +1,7 @@
from setuptools import setup, find_packages from setuptools import setup, find_packages
VERSION = '1.1.0' VERSION = '1.1.0.dev0'
setup( setup(
@ -24,7 +24,8 @@ setup(
packages=find_packages(), packages=find_packages(),
entry_points={ entry_points={
'console_scripts': [ 'console_scripts': [
'borgmatic = borgmatic.command:main', 'borgmatic = borgmatic.commands.borgmatic:main',
'convert-borgmatic-config = borgmatic.commands.convert_config:main',
] ]
}, },
obsoletes=[ obsoletes=[

View file

@ -1,2 +1,5 @@
flexmock==0.10.2 flexmock==0.10.2
pykwalify==1.6.0
pytest==2.9.1 pytest==2.9.1
pytest-cov==2.5.1
ruamel.yaml==0.15.18

View file

@ -5,4 +5,4 @@ skipsdist=True
[testenv] [testenv]
usedevelop=True usedevelop=True
deps=-rtest_requirements.txt deps=-rtest_requirements.txt
commands = py.test borgmatic [] commands = py.test --cov=borgmatic borgmatic []