Merge excludes into config file format.

This commit is contained in:
Dan Helfman 2017-07-10 09:43:25 -07:00
parent 2f7527a333
commit 0dfc935af6
5 changed files with 95 additions and 32 deletions

View file

@ -11,7 +11,6 @@ from borgmatic.config import convert, generate, legacy, validate
DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config' DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
# TODO: Fold excludes into the YAML config file.
DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
@ -21,16 +20,27 @@ def parse_arguments(*arguments):
Given command-line arguments with which this script was invoked, parse the arguments and return Given command-line arguments with which this script was invoked, parse the arguments and return
them as an ArgumentParser instance. them as an ArgumentParser instance.
''' '''
parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.') parser = ArgumentParser(
description='''
Convert legacy INI-style borgmatic configuration and excludes files to a single YAML
configuration file. Note that this replaces any comments from the source files.
'''
)
parser.add_argument( parser.add_argument(
'-s', '--source', '-s', '--source-config',
dest='source_filename', dest='source_config_filename',
default=DEFAULT_SOURCE_CONFIG_FILENAME, default=DEFAULT_SOURCE_CONFIG_FILENAME,
help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME), help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
) )
parser.add_argument( parser.add_argument(
'-d', '--destination', '-e', '--source-excludes',
dest='destination_filename', dest='source_excludes_filename',
default=DEFAULT_SOURCE_EXCLUDES_FILENAME if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME) else None,
help='Excludes filename',
)
parser.add_argument(
'-d', '--destination-config',
dest='destination_config_filename',
default=DEFAULT_DESTINATION_CONFIG_FILENAME, default=DEFAULT_DESTINATION_CONFIG_FILENAME,
help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME), help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
) )
@ -41,12 +51,17 @@ def parse_arguments(*arguments):
def main(): # pragma: no cover def main(): # pragma: no cover
try: try:
args = parse_arguments(*sys.argv[1:]) args = parse_arguments(*sys.argv[1:])
source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT)
schema = yaml.round_trip_load(open(validate.schema_filename()).read()) schema = yaml.round_trip_load(open(validate.schema_filename()).read())
source_config = legacy.parse_configuration(args.source_config_filename, legacy.CONFIG_FORMAT)
source_excludes = (
open(args.source_excludes_filename).read().splitlines()
if args.source_excludes_filename
else []
)
destination_config = convert.convert_legacy_parsed_config(source_config, schema) destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema)
generate.write_configuration(args.destination_filename, destination_config) generate.write_configuration(args.destination_config_filename, destination_config)
# TODO: As a backstop, check that the written config can actually be read and parsed, and # TODO: As a backstop, check that the written config can actually be read and parsed, and
# that it matches the destination config data structure that was written. # that it matches the destination config data structure that was written.

View file

@ -12,16 +12,15 @@ def _convert_section(source_section_config, section_schema):
returned CommentedMap. returned CommentedMap.
''' '''
destination_section_config = yaml.comments.CommentedMap(source_section_config) destination_section_config = yaml.comments.CommentedMap(source_section_config)
generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT)
return destination_section_config return destination_section_config
def convert_legacy_parsed_config(source_config, schema): def convert_legacy_parsed_config(source_config, source_excludes, schema):
''' '''
Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its Given a legacy Parsed_config instance loaded from an INI-style config file and a list of exclude
corresponding yaml.comments.CommentedMap representation in preparation for actual serialization patterns, convert them to a corresponding yaml.comments.CommentedMap representation in
to YAML. preparation for serialization to a single YAML config file.
Additionally, use the given schema as a source of helpful comments to include within the Additionally, use the given schema as a source of helpful comments to include within the
returned CommentedMap. returned CommentedMap.
@ -31,11 +30,21 @@ def convert_legacy_parsed_config(source_config, schema):
for section_name, section_config in source_config._asdict().items() for section_name, section_config in source_config._asdict().items()
]) ])
# Split space-seperated values into actual lists, and merge in excludes.
destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ') destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
destination_config['location']['exclude_patterns'] = source_excludes
if source_config.consistency['checks']: if source_config.consistency['checks']:
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
# Add comments to each section, and then add comments to the fields in each section.
generate.add_comments_to_configuration(destination_config, schema) generate.add_comments_to_configuration(destination_config, schema)
for section_name, section_config in destination_config.items():
generate.add_comments_to_configuration(
section_config,
schema['map'][section_name],
indent=generate.INDENT,
)
return destination_config return destination_config

View file

@ -30,6 +30,18 @@ map:
type: scalar type: scalar
desc: Path to local or remote repository. desc: Path to local or remote repository.
example: user@backupserver:sourcehostname.borg example: user@backupserver:sourcehostname.borg
exclude_patterns:
seq:
- type: scalar
desc: |
Exclude patterns. Any paths matching these patterns are excluded from backups.
Globs are expanded. See
https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-patterns for
details.
example:
- '*.pyc'
- /home/*/.cache
- /etc/ssl
storage: storage:
desc: | desc: |
Repository storage options. See Repository storage options. See

View file

@ -1,5 +1,4 @@
import os import os
import sys
from flexmock import flexmock from flexmock import flexmock
import pytest import pytest
@ -14,7 +13,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults():
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
assert parser.verbosity == None assert parser.verbosity is None
def test_parse_arguments_with_filename_arguments_overrides_defaults(): def test_parse_arguments_with_filename_arguments_overrides_defaults():
@ -24,7 +23,7 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
assert parser.config_filename == 'myconfig' assert parser.config_filename == 'myconfig'
assert parser.excludes_filename == 'myexcludes' assert parser.excludes_filename == 'myexcludes'
assert parser.verbosity == None assert parser.verbosity is None
def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none(): def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
@ -33,8 +32,8 @@ def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_non
parser = module.parse_arguments() parser = module.parse_arguments()
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
assert parser.excludes_filename == None assert parser.excludes_filename is None
assert parser.verbosity == None assert parser.verbosity is None
def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename(): def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename():
@ -44,7 +43,7 @@ def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename(
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
assert parser.excludes_filename == 'myexcludes' assert parser.excludes_filename == 'myexcludes'
assert parser.verbosity == None assert parser.verbosity is None
def test_parse_arguments_with_verbosity_flag_overrides_default(): def test_parse_arguments_with_verbosity_flag_overrides_default():
@ -59,11 +58,6 @@ def test_parse_arguments_with_verbosity_flag_overrides_default():
def test_parse_arguments_with_invalid_arguments_exits(): def test_parse_arguments_with_invalid_arguments_exits():
flexmock(os.path).should_receive('exists').and_return(True) flexmock(os.path).should_receive('exists').and_return(True)
original_stderr = sys.stderr
sys.stderr = sys.stdout
try:
with pytest.raises(SystemExit): with pytest.raises(SystemExit):
module.parse_arguments('--posix-me-harder') module.parse_arguments('--posix-me-harder')
finally:
sys.stderr = original_stderr

View file

@ -1,14 +1,47 @@
import os
from flexmock import flexmock
import pytest
from borgmatic.commands import convert_config as module from borgmatic.commands import convert_config as module
def test_parse_arguments_with_no_arguments_uses_defaults(): def test_parse_arguments_with_no_arguments_uses_defaults():
flexmock(os.path).should_receive('exists').and_return(True)
parser = module.parse_arguments() parser = module.parse_arguments()
assert parser.source_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME assert parser.source_excludes_filename == module.DEFAULT_SOURCE_EXCLUDES_FILENAME
assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
def test_parse_arguments_with_filename_arguments_overrides_defaults(): def test_parse_arguments_with_filename_arguments_overrides_defaults():
parser = module.parse_arguments('--source', 'config', '--destination', 'config.yaml') flexmock(os.path).should_receive('exists').and_return(True)
assert parser.source_filename == 'config' parser = module.parse_arguments(
assert parser.destination_filename == 'config.yaml' '--source-config', 'config',
'--source-excludes', 'excludes',
'--destination-config', 'config.yaml',
)
assert parser.source_config_filename == 'config'
assert parser.source_excludes_filename == 'excludes'
assert parser.destination_config_filename == 'config.yaml'
def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
flexmock(os.path).should_receive('exists').and_return(False)
parser = module.parse_arguments()
assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
assert parser.source_excludes_filename is None
assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
def test_parse_arguments_with_invalid_arguments_exits():
flexmock(os.path).should_receive('exists').and_return(True)
with pytest.raises(SystemExit):
module.parse_arguments('--posix-me-harder')