Basic YAML configuration file parsing.

This commit is contained in:
Dan Helfman 2017-07-04 16:52:24 -07:00
parent 9212f87735
commit 4d7556f68b
16 changed files with 203 additions and 15 deletions

View file

@ -2,6 +2,7 @@ syntax: glob
*.egg-info *.egg-info
*.pyc *.pyc
*.swp *.swp
.cache
.tox .tox
build build
dist dist

1
MANIFEST.in Normal file
View file

@ -0,0 +1 @@
include borgmatic/config/schema.yaml

2
NEWS
View file

@ -1,4 +1,4 @@
1.0.3-dev 1.1.0
* #18: Fix for README mention of sample files not included in package. * #18: Fix for README mention of sample files not included in package.
* #22: Sample files for triggering borgmatic from a systemd timer. * #22: Sample files for triggering borgmatic from a systemd timer.

View file

@ -5,7 +5,7 @@ from subprocess import CalledProcessError
import sys import sys
from borgmatic import borg from borgmatic import borg
from borgmatic.config import parse_configuration, CONFIG_FORMAT from borgmatic.config.legacy import parse_configuration, CONFIG_FORMAT
DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config' DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'

View file

View file

@ -0,0 +1,48 @@
map:
location:
required: True
map:
source_directories:
required: True
seq:
- type: scalar
one_file_system:
type: bool
remote_path:
type: scalar
repository:
required: True
type: scalar
storage:
map:
encryption_passphrase:
type: scalar
compression:
type: scalar
umask:
type: scalar
retention:
map:
keep_within:
type: scalar
keep_hourly:
type: int
keep_daily:
type: int
keep_weekly:
type: int
keep_monthly:
type: int
keep_yearly:
type: int
prefix:
type: scalar
consistency:
map:
checks:
seq:
- type: str
enum: ['repository', 'archives', 'disabled']
unique: True
check_last:
type: int

84
borgmatic/config/yaml.py Normal file
View file

@ -0,0 +1,84 @@
import logging
import sys
import warnings
import pkg_resources
import pykwalify.core
import pykwalify.errors
import ruamel.yaml.error
def schema_filename():
'''
Path to the installed YAML configuration schema file, used to validate and parse the
configuration.
'''
return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
class Validation_error(ValueError):
'''
A collection of error message strings generated when attempting to validate a particular
configurartion file.
'''
def __init__(self, config_filename, error_messages):
self.config_filename = config_filename
self.error_messages = error_messages
def parse_configuration(config_filename, schema_filename):
'''
Given the path to a config filename in YAML format and the path to a schema filename in
pykwalify YAML schema format, return the parsed configuration as a data structure of nested
dicts and lists corresponding to the schema. Example return value:
{'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
have permissions to read the file, or Validation_error if the config does not match the schema.
'''
warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
try:
validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename])
except pykwalify.errors.CoreError as error:
if 'do not exists on disk' in str(error):
raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
if 'Unable to load any data' in str(error):
# If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
# So reach back to the originating exception from ruamel.yaml for something more useful.
raise Validation_error(config_filename, (error.__context__,))
raise
parsed_result = validator.validate(raise_exception=False)
if validator.validation_errors:
raise Validation_error(config_filename, validator.validation_errors)
return parsed_result
def display_validation_error(validation_error):
'''
Given a Validation_error, display its error messages to stderr.
'''
print(
'An error occurred while parsing a configuration file at {}:'.format(
validation_error.config_filename
),
file=sys.stderr,
)
for error in validation_error.error_messages:
print(error, file=sys.stderr)
# FOR TESTING
if __name__ == '__main__':
try:
configuration = parse_configuration('sample/config.yaml', schema_filename())
print(configuration)
except Validation_error as error:
display_validation_error(error)

View file

@ -1,6 +0,0 @@
from flexmock import flexmock
import sys
def builtins_mock():
return flexmock(sys.modules['builtins'])

View file

@ -3,7 +3,7 @@ from io import StringIO
from collections import OrderedDict from collections import OrderedDict
import string import string
from borgmatic import config as module from borgmatic.config import legacy as module
def test_parse_section_options_with_punctuation_should_return_section_options(): def test_parse_section_options_with_punctuation_should_return_section_options():

View file

View file

@ -3,7 +3,7 @@ from collections import OrderedDict
from flexmock import flexmock from flexmock import flexmock
import pytest import pytest
from borgmatic import config as module from borgmatic.config import legacy as module
def test_option_should_create_config_option(): def test_option_should_create_config_option():

View file

@ -1,11 +1,11 @@
from collections import OrderedDict from collections import OrderedDict
from subprocess import STDOUT from subprocess import STDOUT
import sys
import os import os
from flexmock import flexmock from flexmock import flexmock
from borgmatic import borg as module from borgmatic import borg as module
from borgmatic.tests.builtins import builtins_mock
from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
@ -389,7 +389,7 @@ def test_check_archives_should_call_borg_with_parameters():
) )
insert_platform_mock() insert_platform_mock()
insert_datetime_mock() insert_datetime_mock()
builtins_mock().should_receive('open').and_return(stdout) flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull') flexmock(module.os).should_receive('devnull')
module.check_archives( module.check_archives(
@ -464,7 +464,7 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param
) )
insert_platform_mock() insert_platform_mock()
insert_datetime_mock() insert_datetime_mock()
builtins_mock().should_receive('open').and_return(stdout) flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull') flexmock(module.os).should_receive('devnull')
module.check_archives( module.check_archives(

54
sample/config.yaml Normal file
View file

@ -0,0 +1,54 @@
location:
# List of source directories to backup. Globs are expanded.
source_directories:
- /home
- /etc
- /var/log/syslog*
# Stay in same file system (do not cross mount points).
#one_file_system: yes
# Alternate Borg remote executable (defaults to "borg"):
#remote_path: borg1
# Path to local or remote repository.
repository: user@backupserver:sourcehostname.borg
#storage:
# Passphrase to unlock the encryption key with. Only use on repositories
# that were initialized with passphrase/repokey encryption.
#encryption_passphrase: foo
# Type of compression to use when creating archives. See
# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
# for details. Defaults to no compression.
#compression: lz4
# Umask to be used for borg create.
#umask: 0077
retention:
# Retention policy for how many backups to keep in each category. See
# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for
# details.
#keep_within: 3H
#keep_hourly: 24
keep_daily: 7
keep_weekly: 4
keep_monthly: 6
keep_yearly: 1
# When pruning, only consider archive names starting with this prefix.
#prefix: sourcehostname
consistency:
# List of consistency checks to run: "repository", "archives", or both.
# Defaults to both. Set to "disabled" to disable all consistency checks.
# See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
# for details.
checks:
- repository
- archives
# Restrict the number of checked archives to the last n.
#check_last: 3

View file

@ -1,7 +1,7 @@
from setuptools import setup, find_packages from setuptools import setup, find_packages
VERSION = '1.0.3-dev' VERSION = '1.1.0'
setup( setup(
@ -30,8 +30,14 @@ setup(
obsoletes=[ obsoletes=[
'atticmatic', 'atticmatic',
], ],
install_requires=(
'pykwalify',
'ruamel.yaml<=0.15',
'setuptools',
),
tests_require=( tests_require=(
'flexmock', 'flexmock',
'pytest', 'pytest',
) ),
include_package_data=True,
) )