Merge excludes into config file format.
This commit is contained in:
parent
2f7527a333
commit
0dfc935af6
5 changed files with 95 additions and 32 deletions
|
@ -11,7 +11,6 @@ from borgmatic.config import convert, generate, legacy, validate
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
|
DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
|
||||||
# TODO: Fold excludes into the YAML config file.
|
|
||||||
DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
|
DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
|
||||||
DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
|
DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
|
||||||
|
|
||||||
|
@ -21,16 +20,27 @@ def parse_arguments(*arguments):
|
||||||
Given command-line arguments with which this script was invoked, parse the arguments and return
|
Given command-line arguments with which this script was invoked, parse the arguments and return
|
||||||
them as an ArgumentParser instance.
|
them as an ArgumentParser instance.
|
||||||
'''
|
'''
|
||||||
parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.')
|
parser = ArgumentParser(
|
||||||
|
description='''
|
||||||
|
Convert legacy INI-style borgmatic configuration and excludes files to a single YAML
|
||||||
|
configuration file. Note that this replaces any comments from the source files.
|
||||||
|
'''
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-s', '--source',
|
'-s', '--source-config',
|
||||||
dest='source_filename',
|
dest='source_config_filename',
|
||||||
default=DEFAULT_SOURCE_CONFIG_FILENAME,
|
default=DEFAULT_SOURCE_CONFIG_FILENAME,
|
||||||
help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
|
help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME),
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-d', '--destination',
|
'-e', '--source-excludes',
|
||||||
dest='destination_filename',
|
dest='source_excludes_filename',
|
||||||
|
default=DEFAULT_SOURCE_EXCLUDES_FILENAME if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME) else None,
|
||||||
|
help='Excludes filename',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'-d', '--destination-config',
|
||||||
|
dest='destination_config_filename',
|
||||||
default=DEFAULT_DESTINATION_CONFIG_FILENAME,
|
default=DEFAULT_DESTINATION_CONFIG_FILENAME,
|
||||||
help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
|
help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME),
|
||||||
)
|
)
|
||||||
|
@ -41,12 +51,17 @@ def parse_arguments(*arguments):
|
||||||
def main(): # pragma: no cover
|
def main(): # pragma: no cover
|
||||||
try:
|
try:
|
||||||
args = parse_arguments(*sys.argv[1:])
|
args = parse_arguments(*sys.argv[1:])
|
||||||
source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT)
|
|
||||||
schema = yaml.round_trip_load(open(validate.schema_filename()).read())
|
schema = yaml.round_trip_load(open(validate.schema_filename()).read())
|
||||||
|
source_config = legacy.parse_configuration(args.source_config_filename, legacy.CONFIG_FORMAT)
|
||||||
|
source_excludes = (
|
||||||
|
open(args.source_excludes_filename).read().splitlines()
|
||||||
|
if args.source_excludes_filename
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
|
||||||
destination_config = convert.convert_legacy_parsed_config(source_config, schema)
|
destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema)
|
||||||
|
|
||||||
generate.write_configuration(args.destination_filename, destination_config)
|
generate.write_configuration(args.destination_config_filename, destination_config)
|
||||||
|
|
||||||
# TODO: As a backstop, check that the written config can actually be read and parsed, and
|
# TODO: As a backstop, check that the written config can actually be read and parsed, and
|
||||||
# that it matches the destination config data structure that was written.
|
# that it matches the destination config data structure that was written.
|
||||||
|
|
|
@ -12,16 +12,15 @@ def _convert_section(source_section_config, section_schema):
|
||||||
returned CommentedMap.
|
returned CommentedMap.
|
||||||
'''
|
'''
|
||||||
destination_section_config = yaml.comments.CommentedMap(source_section_config)
|
destination_section_config = yaml.comments.CommentedMap(source_section_config)
|
||||||
generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT)
|
|
||||||
|
|
||||||
return destination_section_config
|
return destination_section_config
|
||||||
|
|
||||||
|
|
||||||
def convert_legacy_parsed_config(source_config, schema):
|
def convert_legacy_parsed_config(source_config, source_excludes, schema):
|
||||||
'''
|
'''
|
||||||
Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its
|
Given a legacy Parsed_config instance loaded from an INI-style config file and a list of exclude
|
||||||
corresponding yaml.comments.CommentedMap representation in preparation for actual serialization
|
patterns, convert them to a corresponding yaml.comments.CommentedMap representation in
|
||||||
to YAML.
|
preparation for serialization to a single YAML config file.
|
||||||
|
|
||||||
Additionally, use the given schema as a source of helpful comments to include within the
|
Additionally, use the given schema as a source of helpful comments to include within the
|
||||||
returned CommentedMap.
|
returned CommentedMap.
|
||||||
|
@ -31,11 +30,21 @@ def convert_legacy_parsed_config(source_config, schema):
|
||||||
for section_name, section_config in source_config._asdict().items()
|
for section_name, section_config in source_config._asdict().items()
|
||||||
])
|
])
|
||||||
|
|
||||||
|
# Split space-seperated values into actual lists, and merge in excludes.
|
||||||
destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
|
destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ')
|
||||||
|
destination_config['location']['exclude_patterns'] = source_excludes
|
||||||
|
|
||||||
if source_config.consistency['checks']:
|
if source_config.consistency['checks']:
|
||||||
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
|
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
|
||||||
|
|
||||||
|
# Add comments to each section, and then add comments to the fields in each section.
|
||||||
generate.add_comments_to_configuration(destination_config, schema)
|
generate.add_comments_to_configuration(destination_config, schema)
|
||||||
|
|
||||||
|
for section_name, section_config in destination_config.items():
|
||||||
|
generate.add_comments_to_configuration(
|
||||||
|
section_config,
|
||||||
|
schema['map'][section_name],
|
||||||
|
indent=generate.INDENT,
|
||||||
|
)
|
||||||
|
|
||||||
return destination_config
|
return destination_config
|
||||||
|
|
|
@ -30,6 +30,18 @@ map:
|
||||||
type: scalar
|
type: scalar
|
||||||
desc: Path to local or remote repository.
|
desc: Path to local or remote repository.
|
||||||
example: user@backupserver:sourcehostname.borg
|
example: user@backupserver:sourcehostname.borg
|
||||||
|
exclude_patterns:
|
||||||
|
seq:
|
||||||
|
- type: scalar
|
||||||
|
desc: |
|
||||||
|
Exclude patterns. Any paths matching these patterns are excluded from backups.
|
||||||
|
Globs are expanded. See
|
||||||
|
https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-patterns for
|
||||||
|
details.
|
||||||
|
example:
|
||||||
|
- '*.pyc'
|
||||||
|
- /home/*/.cache
|
||||||
|
- /etc/ssl
|
||||||
storage:
|
storage:
|
||||||
desc: |
|
desc: |
|
||||||
Repository storage options. See
|
Repository storage options. See
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
import os
|
import os
|
||||||
import sys
|
|
||||||
|
|
||||||
from flexmock import flexmock
|
from flexmock import flexmock
|
||||||
import pytest
|
import pytest
|
||||||
|
@ -14,7 +13,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults():
|
||||||
|
|
||||||
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
|
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
|
||||||
assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
|
assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
|
||||||
assert parser.verbosity == None
|
assert parser.verbosity is None
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_filename_arguments_overrides_defaults():
|
def test_parse_arguments_with_filename_arguments_overrides_defaults():
|
||||||
|
@ -24,7 +23,7 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
|
||||||
|
|
||||||
assert parser.config_filename == 'myconfig'
|
assert parser.config_filename == 'myconfig'
|
||||||
assert parser.excludes_filename == 'myexcludes'
|
assert parser.excludes_filename == 'myexcludes'
|
||||||
assert parser.verbosity == None
|
assert parser.verbosity is None
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
|
def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
|
||||||
|
@ -33,8 +32,8 @@ def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_non
|
||||||
parser = module.parse_arguments()
|
parser = module.parse_arguments()
|
||||||
|
|
||||||
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
|
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
|
||||||
assert parser.excludes_filename == None
|
assert parser.excludes_filename is None
|
||||||
assert parser.verbosity == None
|
assert parser.verbosity is None
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename():
|
def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename():
|
||||||
|
@ -44,7 +43,7 @@ def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename(
|
||||||
|
|
||||||
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
|
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
|
||||||
assert parser.excludes_filename == 'myexcludes'
|
assert parser.excludes_filename == 'myexcludes'
|
||||||
assert parser.verbosity == None
|
assert parser.verbosity is None
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_verbosity_flag_overrides_default():
|
def test_parse_arguments_with_verbosity_flag_overrides_default():
|
||||||
|
@ -59,11 +58,6 @@ def test_parse_arguments_with_verbosity_flag_overrides_default():
|
||||||
|
|
||||||
def test_parse_arguments_with_invalid_arguments_exits():
|
def test_parse_arguments_with_invalid_arguments_exits():
|
||||||
flexmock(os.path).should_receive('exists').and_return(True)
|
flexmock(os.path).should_receive('exists').and_return(True)
|
||||||
original_stderr = sys.stderr
|
|
||||||
sys.stderr = sys.stdout
|
|
||||||
|
|
||||||
try:
|
with pytest.raises(SystemExit):
|
||||||
with pytest.raises(SystemExit):
|
module.parse_arguments('--posix-me-harder')
|
||||||
module.parse_arguments('--posix-me-harder')
|
|
||||||
finally:
|
|
||||||
sys.stderr = original_stderr
|
|
||||||
|
|
|
@ -1,14 +1,47 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
from flexmock import flexmock
|
||||||
|
import pytest
|
||||||
|
|
||||||
from borgmatic.commands import convert_config as module
|
from borgmatic.commands import convert_config as module
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_no_arguments_uses_defaults():
|
def test_parse_arguments_with_no_arguments_uses_defaults():
|
||||||
|
flexmock(os.path).should_receive('exists').and_return(True)
|
||||||
|
|
||||||
parser = module.parse_arguments()
|
parser = module.parse_arguments()
|
||||||
|
|
||||||
assert parser.source_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
|
assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
|
||||||
assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
|
assert parser.source_excludes_filename == module.DEFAULT_SOURCE_EXCLUDES_FILENAME
|
||||||
|
assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_filename_arguments_overrides_defaults():
|
def test_parse_arguments_with_filename_arguments_overrides_defaults():
|
||||||
parser = module.parse_arguments('--source', 'config', '--destination', 'config.yaml')
|
flexmock(os.path).should_receive('exists').and_return(True)
|
||||||
|
|
||||||
assert parser.source_filename == 'config'
|
parser = module.parse_arguments(
|
||||||
assert parser.destination_filename == 'config.yaml'
|
'--source-config', 'config',
|
||||||
|
'--source-excludes', 'excludes',
|
||||||
|
'--destination-config', 'config.yaml',
|
||||||
|
)
|
||||||
|
|
||||||
|
assert parser.source_config_filename == 'config'
|
||||||
|
assert parser.source_excludes_filename == 'excludes'
|
||||||
|
assert parser.destination_config_filename == 'config.yaml'
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
|
||||||
|
flexmock(os.path).should_receive('exists').and_return(False)
|
||||||
|
|
||||||
|
parser = module.parse_arguments()
|
||||||
|
|
||||||
|
assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME
|
||||||
|
assert parser.source_excludes_filename is None
|
||||||
|
assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_arguments_with_invalid_arguments_exits():
|
||||||
|
flexmock(os.path).should_receive('exists').and_return(True)
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit):
|
||||||
|
module.parse_arguments('--posix-me-harder')
|
||||||
|
|
Loading…
Reference in a new issue