Basic YAML configuration file parsing.

This commit is contained in:
Dan Helfman 2017-07-04 16:52:24 -07:00
parent 9212f87735
commit 4d7556f68b
16 changed files with 203 additions and 15 deletions

View file

@ -2,6 +2,7 @@ syntax: glob
*.egg-info
*.pyc
*.swp
.cache
.tox
build
dist

1
MANIFEST.in Normal file
View file

@ -0,0 +1 @@
include borgmatic/config/schema.yaml

2
NEWS
View file

@ -1,4 +1,4 @@
1.0.3-dev
1.1.0
* #18: Fix for README mention of sample files not included in package.
* #22: Sample files for triggering borgmatic from a systemd timer.

View file

@ -5,7 +5,7 @@ from subprocess import CalledProcessError
import sys
from borgmatic import borg
from borgmatic.config import parse_configuration, CONFIG_FORMAT
from borgmatic.config.legacy import parse_configuration, CONFIG_FORMAT
DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config'

View file

View file

@ -0,0 +1,48 @@
map:
location:
required: True
map:
source_directories:
required: True
seq:
- type: scalar
one_file_system:
type: bool
remote_path:
type: scalar
repository:
required: True
type: scalar
storage:
map:
encryption_passphrase:
type: scalar
compression:
type: scalar
umask:
type: scalar
retention:
map:
keep_within:
type: scalar
keep_hourly:
type: int
keep_daily:
type: int
keep_weekly:
type: int
keep_monthly:
type: int
keep_yearly:
type: int
prefix:
type: scalar
consistency:
map:
checks:
seq:
- type: str
enum: ['repository', 'archives', 'disabled']
unique: True
check_last:
type: int

84
borgmatic/config/yaml.py Normal file
View file

@ -0,0 +1,84 @@
import logging
import sys
import warnings
import pkg_resources
import pykwalify.core
import pykwalify.errors
import ruamel.yaml.error
def schema_filename():
'''
Path to the installed YAML configuration schema file, used to validate and parse the
configuration.
'''
return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
class Validation_error(ValueError):
'''
A collection of error message strings generated when attempting to validate a particular
configurartion file.
'''
def __init__(self, config_filename, error_messages):
self.config_filename = config_filename
self.error_messages = error_messages
def parse_configuration(config_filename, schema_filename):
'''
Given the path to a config filename in YAML format and the path to a schema filename in
pykwalify YAML schema format, return the parsed configuration as a data structure of nested
dicts and lists corresponding to the schema. Example return value:
{'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
have permissions to read the file, or Validation_error if the config does not match the schema.
'''
warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning)
logging.getLogger('pykwalify').setLevel(logging.CRITICAL)
try:
validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename])
except pykwalify.errors.CoreError as error:
if 'do not exists on disk' in str(error):
raise FileNotFoundError("No such file or directory: '{}'".format(config_filename))
if 'Unable to load any data' in str(error):
# If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful.
# So reach back to the originating exception from ruamel.yaml for something more useful.
raise Validation_error(config_filename, (error.__context__,))
raise
parsed_result = validator.validate(raise_exception=False)
if validator.validation_errors:
raise Validation_error(config_filename, validator.validation_errors)
return parsed_result
def display_validation_error(validation_error):
'''
Given a Validation_error, display its error messages to stderr.
'''
print(
'An error occurred while parsing a configuration file at {}:'.format(
validation_error.config_filename
),
file=sys.stderr,
)
for error in validation_error.error_messages:
print(error, file=sys.stderr)
# FOR TESTING
if __name__ == '__main__':
try:
configuration = parse_configuration('sample/config.yaml', schema_filename())
print(configuration)
except Validation_error as error:
display_validation_error(error)

View file

@ -1,6 +0,0 @@
from flexmock import flexmock
import sys
def builtins_mock():
return flexmock(sys.modules['builtins'])

View file

@ -3,7 +3,7 @@ from io import StringIO
from collections import OrderedDict
import string
from borgmatic import config as module
from borgmatic.config import legacy as module
def test_parse_section_options_with_punctuation_should_return_section_options():

View file

View file

@ -3,7 +3,7 @@ from collections import OrderedDict
from flexmock import flexmock
import pytest
from borgmatic import config as module
from borgmatic.config import legacy as module
def test_option_should_create_config_option():

View file

@ -1,11 +1,11 @@
from collections import OrderedDict
from subprocess import STDOUT
import sys
import os
from flexmock import flexmock
from borgmatic import borg as module
from borgmatic.tests.builtins import builtins_mock
from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
@ -389,7 +389,7 @@ def test_check_archives_should_call_borg_with_parameters():
)
insert_platform_mock()
insert_datetime_mock()
builtins_mock().should_receive('open').and_return(stdout)
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull')
module.check_archives(
@ -464,7 +464,7 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param
)
insert_platform_mock()
insert_datetime_mock()
builtins_mock().should_receive('open').and_return(stdout)
flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull')
module.check_archives(

54
sample/config.yaml Normal file
View file

@ -0,0 +1,54 @@
location:
# List of source directories to backup. Globs are expanded.
source_directories:
- /home
- /etc
- /var/log/syslog*
# Stay in same file system (do not cross mount points).
#one_file_system: yes
# Alternate Borg remote executable (defaults to "borg"):
#remote_path: borg1
# Path to local or remote repository.
repository: user@backupserver:sourcehostname.borg
#storage:
# Passphrase to unlock the encryption key with. Only use on repositories
# that were initialized with passphrase/repokey encryption.
#encryption_passphrase: foo
# Type of compression to use when creating archives. See
# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
# for details. Defaults to no compression.
#compression: lz4
# Umask to be used for borg create.
#umask: 0077
retention:
# Retention policy for how many backups to keep in each category. See
# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for
# details.
#keep_within: 3H
#keep_hourly: 24
keep_daily: 7
keep_weekly: 4
keep_monthly: 6
keep_yearly: 1
# When pruning, only consider archive names starting with this prefix.
#prefix: sourcehostname
consistency:
# List of consistency checks to run: "repository", "archives", or both.
# Defaults to both. Set to "disabled" to disable all consistency checks.
# See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check
# for details.
checks:
- repository
- archives
# Restrict the number of checked archives to the last n.
#check_last: 3

View file

@ -1,7 +1,7 @@
from setuptools import setup, find_packages
VERSION = '1.0.3-dev'
VERSION = '1.1.0'
setup(
@ -30,8 +30,14 @@ setup(
obsoletes=[
'atticmatic',
],
install_requires=(
'pykwalify',
'ruamel.yaml<=0.15',
'setuptools',
),
tests_require=(
'flexmock',
'pytest',
)
),
include_package_data=True,
)