psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Better error messages! Switch the library used for validating configuration files (from pykwalify to jsonschema).

Browse source
This commit is contained in:
Dan Helfman 2021-06-22 13:27:59 -07:00
parent 77a860cc62
commit 27d37b606b
12 changed files with 434 additions and 361 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
NEWS
View file

@ -1,4 +1,6 @@
1.5.16.dev0
* Better error messages! Switch the library used for validating configuration files (from pykwalify
to jsonschema).
* Link borgmatic Ansible role from installation documentation:
https://torsion.org/borgmatic/docs/how-to/set-up-backups/#other-ways-to-install

10
borgmatic/config/convert.py
View file

@ -17,7 +17,7 @@ def _convert_section(source_section_config, section_schema):
(
option_name,
int(option_value)
if section_schema['map'].get(option_name, {}).get('type') == 'int'
if section_schema['properties'].get(option_name, {}).get('type') == 'integer'
else option_value,
)
for option_name, option_value in source_section_config.items()
@ -38,7 +38,7 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
'''
destination_config = yaml.comments.CommentedMap(
[
(section_name, _convert_section(section_config, schema['map'][section_name]))
(section_name, _convert_section(section_config, schema['properties'][section_name]))
for section_name, section_config in source_config._asdict().items()
]
)
@ -54,11 +54,11 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema):
destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
# Add comments to each section, and then add comments to the fields in each section.
generate.add_comments_to_configuration_map(destination_config, schema)
generate.add_comments_to_configuration_object(destination_config, schema)
for section_name, section_config in destination_config.items():
generate.add_comments_to_configuration_map(
section_config, schema['map'][section_name], indent=generate.INDENT
generate.add_comments_to_configuration_object(
section_config, schema['properties'][section_name], indent=generate.INDENT
)
return destination_config

42
borgmatic/config/generate.py
View file

@ -24,29 +24,27 @@ def _insert_newline_before_comment(config, field_name):
def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
'''
Given a loaded configuration schema, generate and return sample config for it. Include comments
for each section based on the schema "desc" description.
for each section based on the schema "description".
'''
schema_type = schema.get('type')
example = schema.get('example')
if example is not None:
return example
if 'seq' in schema:
if schema_type == 'array':
config = yaml.comments.CommentedSeq(
[
_schema_to_sample_configuration(item_schema, level, parent_is_sequence=True)
for item_schema in schema['seq']
]
[_schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)]
)
add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
elif 'map' in schema:
elif schema_type == 'object':
config = yaml.comments.CommentedMap(
[
(field_name, _schema_to_sample_configuration(sub_schema, level + 1))
for field_name, sub_schema in schema['map'].items()
for field_name, sub_schema in schema['properties'].items()
]
)
indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
add_comments_to_configuration_map(
add_comments_to_configuration_object(
config, schema, indent=indent, skip_first=parent_is_sequence
)
else:
@ -132,8 +130,8 @@ def write_configuration(config_filename, rendered_config, mode=0o600):
def add_comments_to_configuration_sequence(config, schema, indent=0):
'''
If the given config sequence's items are maps, then mine the schema for the description of the
map's first item, and slap that atop the sequence. Indent the comment the given number of
If the given config sequence's items are object, then mine the schema for the description of the
object's first item, and slap that atop the sequence. Indent the comment the given number of
characters.
Doing this for sequences of maps results in nice comments that look like:
@ -142,16 +140,16 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
things:
# First key description. Added by this function.
- key: foo
# Second key description. Added by add_comments_to_configuration_map().
# Second key description. Added by add_comments_to_configuration_object().
other: bar
```
'''
if 'map' not in schema['seq'][0]:
if schema['items'].get('type') != 'object':
return
for field_name in config[0].keys():
field_schema = schema['seq'][0]['map'].get(field_name, {})
description = field_schema.get('desc')
field_schema = schema['items']['properties'].get(field_name, {})
description = field_schema.get('description')
# No description to use? Skip it.
if not field_schema or not description:
@ -160,7 +158,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
config[0].yaml_set_start_comment(description, indent=indent)
# We only want the first key's description here, as the rest of the keys get commented by
# add_comments_to_configuration_map().
# add_comments_to_configuration_object().
return
@ -169,7 +167,7 @@ REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
COMMENTED_OUT_SENTINEL = 'COMMENT_OUT'
def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False):
def add_comments_to_configuration_object(config, schema, indent=0, skip_first=False):
'''
Using descriptions from a schema as a source, add those descriptions as comments to the given
config mapping, before each field. Indent the comment the given number of characters.
@ -178,8 +176,8 @@ def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False
if skip_first and index == 0:
continue
field_schema = schema['map'].get(field_name, {})
description = field_schema.get('desc', '').strip()
field_schema = schema['properties'].get(field_name, {})
description = field_schema.get('description', '').strip()
# If this is an optional key, add an indicator to the comment flagging it to be commented
# out from the sample configuration. This sentinel is consumed by downstream processing that
@ -268,9 +266,9 @@ def merge_source_configuration_into_destination(destination_config, source_confi
def generate_sample_configuration(source_filename, destination_filename, schema_filename):
'''
Given an optional source configuration filename, and a required destination configuration
filename, and the path to a schema filename in pykwalify YAML schema format, write out a
sample configuration file based on that schema. If a source filename is provided, merge the
parsed contents of that configuration into the generated configuration.
filename, and the path to a schema filename in a YAML rendition of the JSON Schema format,
write out a sample configuration file based on that schema. If a source filename is provided,
merge the parsed contents of that configuration into the generated configuration.
'''
schema = yaml.round_trip_load(open(schema_filename))
source_config = None

517
borgmatic/config/schema.yaml
View file

@ -1,19 +1,25 @@
name: Borgmatic configuration file schema
version: 1
map:
type: object
required:
- location
additionalProperties: false
properties:
location:
desc: |
type: object
description: |
Where to look for files to backup, and where to store those backups.
See https://borgbackup.readthedocs.io/en/stable/quickstart.html and
https://borgbackup.readthedocs.io/en/stable/usage/create.html
for details.
required: true
map:
required:
- source_directories
- repositories
additionalProperties: false
properties:
source_directories:
required: true
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of source directories to backup (required). Globs and
tildes are expanded. Do not backslash spaces in path names.
example:
@ -22,10 +28,10 @@ map:
- /var/log/syslog*
- /home/user/path with spaces
repositories:
required: true
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
Paths to local or remote repositories (required). Tildes are
expanded. Multiple repositories are backed up to in
sequence. Borg placeholders can be used. See the output of
@ -37,36 +43,36 @@ map:
- user@backupserver:sourcehostname.borg
- "user@backupserver:{fqdn}"
one_file_system:
type: bool
desc: |
type: boolean
description: |
Stay in same file system (do not cross mount points).
Defaults to false. But when a database hook is used, the
setting here is ignored and one_file_system is considered
true.
example: true
numeric_owner:
type: bool
desc: |
type: boolean
description: |
Only store/extract numeric user and group identifiers.
Defaults to false.
example: true
atime:
type: bool
desc: Store atime into archive. Defaults to true.
type: boolean
description: Store atime into archive. Defaults to true.
example: false
ctime:
type: bool
desc: Store ctime into archive. Defaults to true.
type: boolean
description: Store ctime into archive. Defaults to true.
example: false
birthtime:
type: bool
desc: |
type: boolean
description: |
Store birthtime (creation date) into archive. Defaults to
true.
example: false
read_special:
type: bool
desc: |
type: boolean
description: |
Use Borg's --read-special flag to allow backup of block and
other special devices. Use with caution, as it will lead to
problems if used when backing up special devices such as
@ -75,30 +81,33 @@ map:
considered true.
example: false
bsd_flags:
type: bool
desc: |
type: boolean
description: |
Record bsdflags (e.g. NODUMP, IMMUTABLE) in archive.
Defaults to true.
example: true
files_cache:
type: str
desc: |
type: string
description: |
Mode in which to operate the files cache. See
http://borgbackup.readthedocs.io/en/stable/usage/create.html
for details. Defaults to "ctime,size,inode".
example: ctime,size,inode
local_path:
type: str
desc: Alternate Borg local executable. Defaults to "borg".
type: string
description: |
Alternate Borg local executable. Defaults to "borg".
example: borg1
remote_path:
type: str
desc: Alternate Borg remote executable. Defaults to "borg".
type: string
description: |
Alternate Borg remote executable. Defaults to "borg".
example: borg1
patterns:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
Any paths matching these patterns are included/excluded from
backups. Globs are expanded. (Tildes are not.) Note that
Borg considers this option experimental. See the output of
@ -110,9 +119,10 @@ map:
- '+ /home/susan'
- '- /home/*'
patterns_from:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
Read include/exclude patterns from one or more separate
named files, one pattern per line. Note that Borg considers
this option experimental. See the output of "borg help
@ -120,9 +130,10 @@ map:
example:
- /etc/borgmatic/patterns
exclude_patterns:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
Any paths matching these patterns are excluded from backups.
Globs and tildes are expanded. Do not backslash spaces in
path names. See the output of "borg help patterns" for more
@ -133,59 +144,63 @@ map:
- /etc/ssl
- /home/user/path with spaces
exclude_from:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
Read exclude patterns from one or more separate named files,
one pattern per line. See the output of "borg help patterns"
for more details.
example:
- /etc/borgmatic/excludes
exclude_caches:
type: bool
desc: |
type: boolean
description: |
Exclude directories that contain a CACHEDIR.TAG file. See
http://www.brynosaurus.com/cachedir/spec.html for details.
Defaults to false.
example: true
exclude_if_present:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
Exclude directories that contain a file with the given
filenames. Defaults to not set.
example:
- .nobackup
keep_exclude_tags:
type: bool
desc: |
type: boolean
description: |
If true, the exclude_if_present filename is included in
backups. Defaults to false, meaning that the
exclude_if_present filename is omitted from backups.
example: true
exclude_nodump:
type: bool
desc: |
type: boolean
description: |
Exclude files with the NODUMP flag. Defaults to false.
example: true
borgmatic_source_directory:
type: str
desc: |
type: string
description: |
Path for additional source files used for temporary internal
state like borgmatic database dumps. Note that changing this
path prevents "borgmatic restore" from finding any database
dumps created before the change. Defaults to ~/.borgmatic
example: /tmp/borgmatic
storage:
desc: |
type: object
description: |
Repository storage options. See
https://borgbackup.readthedocs.io/en/stable/usage/create.html and
https://borgbackup.readthedocs.io/en/stable/usage/general.html for
details.
map:
additionalProperties: false
properties:
encryption_passcommand:
type: str
desc: |
type: string
description: |
The standard output of this command is used to unlock the
encryption key. Only use on repositories that were
initialized with passcommand/repokey/keyfile encryption.
@ -194,8 +209,8 @@ map:
takes precedence. Defaults to not set.
example: "secret-tool lookup borg-repository repo-name"
encryption_passphrase:
type: str
desc: |
type: string
description: |
Passphrase to unlock the encryption key with. Only use on
repositories that were initialized with
passphrase/repokey/keyfile encryption. Quote the value if it
@ -204,8 +219,8 @@ map:
set.
example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
checkpoint_interval:
type: int
desc: |
type: integer
description: |
Number of seconds between each checkpoint during a
long-running backup. See
https://borgbackup.readthedocs.io/en/stable/faq.html
@ -213,8 +228,8 @@ map:
minutes).
example: 1800
chunker_params:
type: str
desc: |
type: string
description: |
Specify the parameters passed to then chunker
(CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS,
HASH_WINDOW_SIZE). See
@ -222,73 +237,73 @@ map:
for details. Defaults to "19,23,21,4095".
example: 19,23,21,4095
compression:
type: str
desc: |
type: string
description: |
Type of compression to use when creating archives. See
http://borgbackup.readthedocs.io/en/stable/usage/create.html
for details. Defaults to "lz4".
example: lz4
remote_rate_limit:
type: int
desc: |
type: integer
description: |
Remote network upload rate limit in kiBytes/second. Defaults
to unlimited.
example: 100
temporary_directory:
type: str
desc: |
type: string
description: |
Directory where temporary files are stored. Defaults to
$TMPDIR
example: /path/to/tmpdir
ssh_command:
type: str
desc: |
type: string
description: |
Command to use instead of "ssh". This can be used to specify
ssh options. Defaults to not set.
example: ssh -i /path/to/private/key
borg_base_directory:
type: str
desc: |
type: string
description: |
Base path used for various Borg directories. Defaults to
$HOME, ~$USER, or ~.
example: /path/to/base
borg_config_directory:
type: str
desc: |
type: string
description: |
Path for Borg configuration files. Defaults to
$borg_base_directory/.config/borg
example: /path/to/base/config
borg_cache_directory:
type: str
desc: |
type: string
description: |
Path for Borg cache files. Defaults to
$borg_base_directory/.cache/borg
example: /path/to/base/cache
borg_security_directory:
type: str
desc: |
type: string
description: |
Path for Borg security and encryption nonce files. Defaults
to $borg_base_directory/.config/borg/security
example: /path/to/base/config/security
borg_keys_directory:
type: str
desc: |
type: string
description: |
Path for Borg encryption key files. Defaults to
$borg_base_directory/.config/borg/keys
example: /path/to/base/config/keys
umask:
type: scalar
desc: Umask to be used for borg create. Defaults to 0077.
type: string
description: Umask to be used for borg create. Defaults to 0077.
example: 0077
lock_wait:
type: int
desc: |
type: integer
description: |
Maximum seconds to wait for acquiring a repository/cache
lock. Defaults to 1.
example: 5
archive_name_format:
type: str
desc: |
type: string
description: |
Name of the archive. Borg placeholders can be used. See the
output of "borg help placeholders" for details. Defaults to
"{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this
@ -298,40 +313,42 @@ map:
prefix in the consistency section as well.
example: "{hostname}-documents-{now}"
relocated_repo_access_is_ok:
type: bool
desc: |
type: boolean
description: |
Bypass Borg error about a repository that has been moved.
Defaults to false.
example: true
unknown_unencrypted_repo_access_is_ok:
type: bool
desc: |
type: boolean
description: |
Bypass Borg error about a previously unknown unencrypted
repository. Defaults to false.
example: true
extra_borg_options:
map:
type: object
additionalProperties: false
properties:
init:
type: str
desc: |
type: string
description: |
Extra command-line options to pass to "borg init".
example: "--make-parent-dirs"
prune:
type: str
desc: |
type: string
description: |
Extra command-line options to pass to "borg prune".
example: "--save-space"
create:
type: str
desc: |
type: string
description: |
Extra command-line options to pass to "borg create".
example: "--no-files-cache"
check:
type: str
desc: |
type: string
description: |
Extra command-line options to pass to "borg check".
example: "--save-space"
desc: |
description: |
Additional options to pass directly to particular Borg
commands, handy for Borg options that borgmatic does not yet
support natively. Note that borgmatic does not perform any
@ -339,72 +356,76 @@ map:
"--verbosity 2" shows the exact Borg command-line
invocation.
retention:
desc: |
type: object
description: |
Retention policy for how many backups to keep in each category. See
https://borgbackup.readthedocs.io/en/stable/usage/prune.html for
details. At least one of the "keep" options is required for pruning
to work. To skip pruning entirely, run "borgmatic create" or "check"
without the "prune" action. See borgmatic documentation for details.
map:
additionalProperties: false
properties:
keep_within:
type: str
desc: Keep all archives within this time interval.
type: string
description: Keep all archives within this time interval.
example: 3H
keep_secondly:
type: int
desc: Number of secondly archives to keep.
type: integer
description: Number of secondly archives to keep.
example: 60
keep_minutely:
type: int
desc: Number of minutely archives to keep.
type: integer
description: Number of minutely archives to keep.
example: 60
keep_hourly:
type: int
desc: Number of hourly archives to keep.
type: integer
description: Number of hourly archives to keep.
example: 24
keep_daily:
type: int
desc: Number of daily archives to keep.
type: integer
description: Number of daily archives to keep.
example: 7
keep_weekly:
type: int
desc: Number of weekly archives to keep.
type: integer
description: Number of weekly archives to keep.
example: 4
keep_monthly:
type: int
desc: Number of monthly archives to keep.
type: integer
description: Number of monthly archives to keep.
example: 6
keep_yearly:
type: int
desc: Number of yearly archives to keep.
type: integer
description: Number of yearly archives to keep.
example: 1
prefix:
type: str
desc: |
type: string
description: |
When pruning, only consider archive names starting with this
prefix. Borg placeholders can be used. See the output of
"borg help placeholders" for details. Defaults to
"{hostname}-". Use an empty value to disable the default.
example: sourcehostname
consistency:
desc: |
type: object
description: |
Consistency checks to run after backups. See
https://borgbackup.readthedocs.io/en/stable/usage/check.html and
https://borgbackup.readthedocs.io/en/stable/usage/extract.html for
details.
map:
additionalProperties: false
properties:
checks:
seq:
- type: str
enum: [
'repository',
'archives',
'data',
'extract',
'disabled'
]
unique: true
desc: |
type: array
items:
type: string
enum:
- repository
- archives
- data
- extract
- disabled
uniqueItems: true
description: |
List of one or more consistency checks to run: "repository",
"archives", "data", and/or "extract". Defaults to
"repository" and "archives". Set to "disabled" to disable
@ -417,9 +438,10 @@ map:
- repository
- archives
check_repositories:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
Paths to a subset of the repositories in the location
section on which to run consistency checks. Handy in case
some of your repositories are very large, and so running
@ -429,15 +451,15 @@ map:
example:
- user@backupserver:sourcehostname.borg
check_last:
type: int
desc: |
type: integer
description: |
Restrict the number of checked archives to the last n.
Applies only to the "archives" check. Defaults to checking
all archives.
example: 3
prefix:
type: str
desc: |
type: string
description: |
When performing the "archives" check, only consider archive
names starting with this prefix. Borg placeholders can be
used. See the output of "borg help placeholders" for
@ -445,101 +467,115 @@ map:
disable the default.
example: sourcehostname
output:
desc: |
type: object
description: |
Options for customizing borgmatic's own output and logging.
map:
additionalProperties: false
properties:
color:
type: bool
desc: |
type: boolean
description: |
Apply color to console output. Can be overridden with
--no-color command-line flag. Defaults to true.
example: false
hooks:
desc: |
type: object
description: |
Shell commands, scripts, or integrations to execute at various
points during a borgmatic run. IMPORTANT: All provided commands and
scripts are executed with user permissions of borgmatic. Do not
forget to set secure permissions on this configuration file (chmod
0600) as well as on any script called from a hook (chmod 0700) to
prevent potential shell injection or privilege escalation.
map:
additionalProperties: false
properties:
before_backup:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
before creating a backup, run once per configuration file.
example:
- echo "Starting a backup."
before_prune:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
before pruning, run once per configuration file.
example:
- echo "Starting pruning."
before_check:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
before consistency checks, run once per configuration file.
example:
- echo "Starting checks."
before_extract:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
before extracting a backup, run once per configuration file.
example:
- echo "Starting extracting."
after_backup:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
after creating a backup, run once per configuration file.
example:
- echo "Finished a backup."
after_prune:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
after pruning, run once per configuration file.
example:
- echo "Finished pruning."
after_check:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
after consistency checks, run once per configuration file.
example:
- echo "Finished checks."
after_extract:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
after extracting a backup, run once per configuration file.
example:
- echo "Finished extracting."
on_error:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
when an exception occurs during a "prune", "create", or
"check" action or an associated before/after hook.
example:
- echo "Error during prune/create/check."
before_everything:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
before running all actions (if one of them is "create").
These are collected from all configuration files and then
@ -547,9 +583,10 @@ map:
example:
- echo "Starting actions."
after_everything:
seq:
- type: str
desc: |
type: array
items:
type: string
description: |
List of one or more shell commands or scripts to execute
after running all actions (if one of them is "create").
These are collected from all configuration files and then
@ -557,12 +594,15 @@ map:
example:
- echo "Completed actions."
postgresql_databases:
seq:
- map:
type: array
items:
type: object
required: ['name']
additionalProperties: false
properties:
name:
required: true
type: str
desc: |
type: string
description: |
Database name (required if using this hook). Or
"all" to dump all databases on the host. Note
that using this database hook implicitly enables
@ -570,26 +610,26 @@ map:
above) to support dump and restore streaming.
example: users
hostname:
type: str
desc: |
type: string
description: |
Database hostname to connect to. Defaults to
connecting via local Unix socket.
example: database.example.org
port:
type: int
desc: Port to connect to. Defaults to 5432.
type: integer
description: Port to connect to. Defaults to 5432.
example: 5433
username:
type: str
desc: |
type: string
description: |
Username with which to connect to the database.
Defaults to the username of the current user.
You probably want to specify the "postgres"
superuser here when the database name is "all".
example: dbuser
password:
type: str
desc: |
type: string
description: |
Password with which to connect to the database.
Omitting a password will only work if PostgreSQL
is configured to trust the configured username
@ -597,9 +637,9 @@ map:
file.
example: trustsome1
format:
type: str
type: string
enum: ['plain', 'custom', 'directory', 'tar']
desc: |
description: |
Database dump output format. One of "plain",
"custom", "directory", or "tar". Defaults to
"custom" (unlike raw pg_dump). See pg_dump
@ -607,45 +647,45 @@ map:
ignored when the database name is "all".
example: directory
ssl_mode:
type: str
type: string
enum: ['disable', 'allow', 'prefer',
'require', 'verify-ca', 'verify-full']
desc: |
description: |
SSL mode to use to connect to the database
server. One of "disable", "allow", "prefer",
"require", "verify-ca" or "verify-full".
Defaults to "disable".
example: require
ssl_cert:
type: str
desc: |
type: string
description: |
Path to a client certificate.
example: "/root/.postgresql/postgresql.crt"
ssl_key:
type: str
desc: |
type: string
description: |
Path to a private client key.
example: "/root/.postgresql/postgresql.key"
ssl_root_cert:
type: str
desc: |
type: string
description: |
Path to a root certificate containing a list of
trusted certificate authorities.
example: "/root/.postgresql/root.crt"
ssl_crl:
type: str
desc: |
type: string
description: |
Path to a certificate revocation list.
example: "/root/.postgresql/root.crl"
options:
type: str
desc: |
type: string
description: |
Additional pg_dump/pg_dumpall options to pass
directly to the dump command, without performing
any validation on them. See pg_dump
documentation for details.
example: --role=someone
desc: |
description: |
List of one or more PostgreSQL databases to dump before
creating a backup, run once per configuration file. The
database dumps are added to your source directories at
@ -655,12 +695,15 @@ map:
https://www.postgresql.org/docs/current/libpq-ssl.html for
details.
mysql_databases:
seq:
- map:
type: array
items:
type: object
required: ['name']
additionalProperties: false
properties:
name:
required: true
type: str
desc: |
type: string
description: |
Database name (required if using this hook). Or
"all" to dump all databases on the host. Note
that using this database hook implicitly enables
@ -668,38 +711,38 @@ map:
above) to support dump and restore streaming.
example: users
hostname:
type: str
desc: |
type: string
description: |
Database hostname to connect to. Defaults to
connecting via local Unix socket.
example: database.example.org
port:
type: int
desc: Port to connect to. Defaults to 3306.
type: integer
description: Port to connect to. Defaults to 3306.
example: 3307
username:
type: str
desc: |
type: string
description: |
Username with which to connect to the database.
Defaults to the username of the current user.
example: dbuser
password:
type: str
desc: |
type: string
description: |
Password with which to connect to the database.
Omitting a password will only work if MySQL is
configured to trust the configured username
without a password.
example: trustsome1
options:
type: str
desc: |
type: string
description: |
Additional mysqldump options to pass directly to
the dump command, without performing any
validation on them. See mysqldump documentation
for details.
example: --skip-comments
desc: |
description: |
List of one or more MySQL/MariaDB databases to dump before
creating a backup, run once per configuration file. The
database dumps are added to your source directories at
@ -708,8 +751,8 @@ map:
https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
https://mariadb.com/kb/en/library/mysqldump/ for details.
healthchecks:
type: str
desc: |
type: string
description: |
Healthchecks ping URL or UUID to notify when a backup
begins, ends, or errors. Create an account at
https://healthchecks.io if you'd like to use this service.
@ -717,8 +760,8 @@ map:
example:
https://hc-ping.com/your-uuid-here
cronitor:
type: str
desc: |
type: string
description: |
Cronitor ping URL to notify when a backup begins, ends, or
errors. Create an account at https://cronitor.io if you'd
like to use this service. See borgmatic monitoring
@ -726,8 +769,8 @@ map:
example:
https://cronitor.link/d3x0c1
pagerduty:
type: str
desc: |
type: string
description: |
PagerDuty integration key used to notify PagerDuty when a
backup errors. Create an account at
https://www.pagerduty.com/ if you'd like to use this
@ -735,8 +778,8 @@ map:
example:
a177cad45bd374409f78906a810a3074
cronhub:
type: str
desc: |
type: string
description: |
Cronhub ping URL to notify when a backup begins, ends, or
errors. Create an account at https://cronhub.io if you'd
like to use this service. See borgmatic monitoring
@ -745,7 +788,7 @@ map:
https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d01
umask:
type: scalar
desc: |
description: |
Umask used when executing hooks. Defaults to the umask that
borgmatic is run with.
example: 0077

78
borgmatic/config/validate.py
View file

@ -1,9 +1,7 @@
import logging
import os
import jsonschema
import pkg_resources
import pykwalify.core
import pykwalify.errors
import ruamel.yaml
from borgmatic.config import load, normalize, override
@ -17,15 +15,40 @@ def schema_filename():
return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
def format_error_path_element(path_element):
'''
Given a path element into a JSON data structure, format it for display as a string.
'''
if isinstance(path_element, int):
return str('[{}]'.format(path_element))
return str('.{}'.format(path_element))
def format_error(error):
'''
Given an instance of jsonschema.exceptions.ValidationError, format it for display as a string.
'''
if not error.path:
return 'At the top level: {}'.format(error.message)
formatted_path = ''.join(format_error_path_element(element) for element in error.path)
return "At '{}': {}".format(formatted_path.lstrip('.'), error.message)
class Validation_error(ValueError):
'''
A collection of error message strings generated when attempting to validate a particular
configurartion file.
A collection of error messages generated when attempting to validate a particular
configuration file.
'''
def __init__(self, config_filename, error_messages):
def __init__(self, config_filename, errors):
'''
Given a configuration filename path and a sequence of
jsonschema.exceptions.ValidationError instances, create a Validation_error.
'''
self.config_filename = config_filename
self.error_messages = error_messages
self.errors = errors
def __str__(self):
'''
@ -33,7 +56,7 @@ class Validation_error(ValueError):
'''
return 'An error occurred while parsing a configuration file at {}:\n'.format(
self.config_filename
) + '\n'.join(self.error_messages)
) + '\n'.join(format_error(error) for error in self.errors)
def apply_logical_validation(config_filename, parsed_configuration):
@ -65,29 +88,12 @@ def apply_logical_validation(config_filename, parsed_configuration):
)
def remove_examples(schema):
'''
pykwalify gets angry if the example field is not a string. So rather than bend to its will,
remove all examples from the given schema before passing the schema to pykwalify.
'''
if 'map' in schema:
for item_name, item_schema in schema['map'].items():
item_schema.pop('example', None)
remove_examples(item_schema)
elif 'seq' in schema:
for item_schema in schema['seq']:
item_schema.pop('example', None)
remove_examples(item_schema)
return schema
def parse_configuration(config_filename, schema_filename, overrides=None):
'''
Given the path to a config filename in YAML format, the path to a schema filename in pykwalify
YAML schema format, a sequence of configuration file override strings in the form of
"section.option=value", return the parsed configuration as a data structure of nested dicts and
lists corresponding to the schema. Example return value:
Given the path to a config filename in YAML format, the path to a schema filename in a YAML
rendition of JSON Schema format, a sequence of configuration file override strings in the form
of "section.option=value", return the parsed configuration as a data structure of nested dicts
and lists corresponding to the schema. Example return value:
{'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
@ -95,8 +101,6 @@ def parse_configuration(config_filename, schema_filename, overrides=None):
Raise FileNotFoundError if the file does not exist, PermissionError if the user does not
have permissions to read the file, or Validation_error if the config does not match the schema.
'''
logging.getLogger('pykwalify').setLevel(logging.ERROR)
try:
config = load.load_configuration(config_filename)
schema = load.load_configuration(schema_filename)
@ -106,15 +110,15 @@ def parse_configuration(config_filename, schema_filename, overrides=None):
override.apply_overrides(config, overrides)
normalize.normalize(config)
validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
parsed_result = validator.validate(raise_exception=False)
validator = jsonschema.Draft7Validator(schema)
validation_errors = tuple(validator.iter_errors(config))
if validator.validation_errors:
raise Validation_error(config_filename, validator.validation_errors)
if validation_errors:
raise Validation_error(config_filename, validation_errors)
apply_logical_validation(config_filename, parsed_result)
apply_logical_validation(config_filename, config)
return parsed_result
return config
def normalize_repository_path(repository):

1
borgmatic/logger.py
View file

@ -1,4 +1,5 @@
import logging
import logging.handlers
import os
import sys

2
setup.py
View file

@ -30,7 +30,7 @@ setup(
},
obsoletes=['atticmatic'],
install_requires=(
'pykwalify>=1.6.0,<14.06',
'jsonschema',
'requests',
'ruamel.yaml>0.15.0,<0.18.0',
'setuptools',

9
test_requirements.txt
View file

@ -1,26 +1,21 @@
appdirs==1.4.4; python_version >= '3.8'
atomicwrites==1.4.0
attrs==20.3.0; python_version >= '3.8'
black==19.10b0; python_version >= '3.8'
click==7.1.2; python_version >= '3.8'
colorama==0.4.4
coverage==5.3
docopt==0.6.2
flake8==3.8.4
flexmock==0.10.4
isort==5.6.4
isort==5.9.1
mccabe==0.6.1
more-itertools==8.6.0
pluggy==0.13.1
pathspec==0.8.1; python_version >= '3.8'
py==1.10.0
pycodestyle==2.6.0
pyflakes==2.2.0
pykwalify==1.7.0
jsonschema==3.2.0
pytest==6.1.2
pytest-cov==2.10.1
python-dateutil==2.8.1
PyYAML==5.4.1
regex; python_version >= '3.8'
requests==2.25.0
ruamel.yaml>0.15.0,<0.18.0

24
tests/integration/config/test_generate.py
View file

@ -122,38 +122,44 @@ def test_write_configuration_with_already_existing_directory_does_not_raise():
def test_add_comments_to_configuration_sequence_of_strings_does_not_raise():
config = module.yaml.comments.CommentedSeq(['foo', 'bar'])
schema = {'seq': [{'type': 'str'}]}
schema = {'type': 'array', 'items': {'type': 'string'}}
module.add_comments_to_configuration_sequence(config, schema)
def test_add_comments_to_configuration_sequence_of_maps_does_not_raise():
config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
schema = {'seq': [{'map': {'foo': {'desc': 'yo'}}}]}
schema = {
'type': 'array',
'items': {'type': 'object', 'properties': {'foo': {'description': 'yo'}}},
}
module.add_comments_to_configuration_sequence(config, schema)
def test_add_comments_to_configuration_sequence_of_maps_without_description_does_not_raise():
config = module.yaml.comments.CommentedSeq([module.yaml.comments.CommentedMap([('foo', 'yo')])])
schema = {'seq': [{'map': {'foo': {}}}]}
schema = {'type': 'array', 'items': {'type': 'object', 'properties': {'foo': {}}}}
module.add_comments_to_configuration_sequence(config, schema)
def test_add_comments_to_configuration_map_does_not_raise():
def test_add_comments_to_configuration_object_does_not_raise():
# Ensure that it can deal with fields both in the schema and missing from the schema.
config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)])
schema = {'map': {'foo': {'desc': 'Foo'}, 'bar': {'desc': 'Bar'}}}
schema = {
'type': 'object',
'properties': {'foo': {'description': 'Foo'}, 'bar': {'description': 'Bar'}},
}
module.add_comments_to_configuration_map(config, schema)
module.add_comments_to_configuration_object(config, schema)
def test_add_comments_to_configuration_map_with_skip_first_does_not_raise():
def test_add_comments_to_configuration_object_with_skip_first_does_not_raise():
config = module.yaml.comments.CommentedMap([('foo', 33)])
schema = {'map': {'foo': {'desc': 'Foo'}}}
schema = {'type': 'object', 'properties': {'foo': {'description': 'Foo'}}}
module.add_comments_to_configuration_map(config, schema, skip_first=True)
module.add_comments_to_configuration_object(config, schema, skip_first=True)
def test_remove_commented_out_sentinel_keeps_other_comments():

16
tests/unit/config/test_convert.py
View file

@ -12,7 +12,7 @@ Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention',
def test_convert_section_generates_integer_value_for_integer_type_in_schema():
flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
source_section_config = OrderedDict([('check_last', '3')])
section_schema = {'map': {'check_last': {'type': 'int'}}}
section_schema = {'type': 'object', 'properties': {'check_last': {'type': 'integer'}}}
destination_config = module._convert_section(source_section_config, section_schema)
@ -21,7 +21,7 @@ def test_convert_section_generates_integer_value_for_integer_type_in_schema():
def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
flexmock(module.generate).should_receive('add_comments_to_configuration_map')
flexmock(module.generate).should_receive('add_comments_to_configuration_object')
source_config = Parsed_config(
location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]),
storage=OrderedDict([('encryption_passphrase', 'supersecret')]),
@ -29,7 +29,10 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
consistency=OrderedDict([('checks', 'repository')]),
)
source_excludes = ['/var']
schema = {'map': defaultdict(lambda: {'map': {}})}
schema = {
'type': 'object',
'properties': defaultdict(lambda: {'type': 'object', 'properties': {}}),
}
destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema)
@ -54,7 +57,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
def test_convert_legacy_parsed_config_splits_space_separated_values():
flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
flexmock(module.generate).should_receive('add_comments_to_configuration_map')
flexmock(module.generate).should_receive('add_comments_to_configuration_object')
source_config = Parsed_config(
location=OrderedDict(
[('source_directories', '/home /etc'), ('repository', 'hostname.borg')]
@ -64,7 +67,10 @@ def test_convert_legacy_parsed_config_splits_space_separated_values():
consistency=OrderedDict([('checks', 'repository archives')]),
)
source_excludes = ['/var']
schema = {'map': defaultdict(lambda: {'map': {}})}
schema = {
'type': 'object',
'properties': defaultdict(lambda: {'type': 'object', 'properties': {}}),
}
destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema)

36
tests/unit/config/test_generate.py
View file

@ -8,24 +8,32 @@ from borgmatic.config import generate as module
def test_schema_to_sample_configuration_generates_config_map_with_examples():
flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
flexmock(module).should_receive('add_comments_to_configuration_map')
flexmock(module).should_receive('add_comments_to_configuration_object')
schema = {
'map': OrderedDict(
'type': 'object',
'properties': OrderedDict(
[
('section1', {'map': {'field1': OrderedDict([('example', 'Example 1')])}}),
(
'section1',
{
'type': 'object',
'properties': {'field1': OrderedDict([('example', 'Example 1')])},
},
),
(
'section2',
{
'map': OrderedDict(
'type': 'object',
'properties': OrderedDict(
[
('field2', {'example': 'Example 2'}),
('field3', {'example': 'Example 3'}),
]
)
),
},
),
]
)
),
}
config = module._schema_to_sample_configuration(schema)
@ -41,7 +49,7 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples():
def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example():
flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
flexmock(module).should_receive('add_comments_to_configuration_sequence')
schema = {'seq': [{'type': 'str'}], 'example': ['hi']}
schema = {'type': 'array', 'items': {'type': 'string'}, 'example': ['hi']}
config = module._schema_to_sample_configuration(schema)
@ -51,15 +59,15 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_strings_wit
def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_examples():
flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
flexmock(module).should_receive('add_comments_to_configuration_sequence')
flexmock(module).should_receive('add_comments_to_configuration_map')
flexmock(module).should_receive('add_comments_to_configuration_object')
schema = {
'seq': [
{
'map': OrderedDict(
'type': 'array',
'items': {
'type': 'object',
'properties': OrderedDict(
[('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})]
)
}
]
),
},
}
config = module._schema_to_sample_configuration(schema)

56
tests/unit/config/test_validate.py
View file

@ -4,8 +4,33 @@ from flexmock import flexmock
from borgmatic.config import validate as module
def test_validation_error_str_contains_error_messages_and_config_filename():
error = module.Validation_error('config.yaml', ('oops', 'uh oh'))
def test_format_error_path_element_formats_array_index():
module.format_error_path_element(3) == '[3]'
def test_format_error_path_element_formats_property():
module.format_error_path_element('foo') == '.foo'
def test_format_error_formats_error_including_path():
flexmock(module).format_error_path_element = lambda element: '.{}'.format(element)
error = flexmock(message='oops', path=['foo', 'bar'])
assert module.format_error(error) == "At 'foo.bar': oops"
def test_format_error_formats_error_without_path():
flexmock(module).should_receive('format_error_path_element').never()
error = flexmock(message='oops', path=[])
assert module.format_error(error) == 'At the top level: oops'
def test_validation_error_string_contains_error_messages_and_config_filename():
flexmock(module).format_error = lambda error: error.message
error = module.Validation_error(
'config.yaml', (flexmock(message='oops', path=None), flexmock(message='uh oh'))
)
result = str(error)
@ -15,6 +40,8 @@ def test_validation_error_str_contains_error_messages_and_config_filename():
def test_apply_logical_validation_raises_if_archive_name_format_present_without_prefix():
flexmock(module).format_error = lambda error: error.message
with pytest.raises(module.Validation_error):
module.apply_logical_validation(
'config.yaml',
@ -26,6 +53,8 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_
def test_apply_logical_validation_raises_if_archive_name_format_present_without_retention_prefix():
flexmock(module).format_error = lambda error: error.message
with pytest.raises(module.Validation_error):
module.apply_logical_validation(
'config.yaml',
@ -38,6 +67,8 @@ def test_apply_logical_validation_raises_if_archive_name_format_present_without_
def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories():
flexmock(module).format_error = lambda error: error.message
with pytest.raises(module.Validation_error):
module.apply_logical_validation(
'config.yaml',
@ -75,27 +106,6 @@ def test_apply_logical_validation_does_not_raise_otherwise():
module.apply_logical_validation('config.yaml', {'retention': {'keep_secondly': 1000}})
def test_remove_examples_strips_examples_from_map():
schema = {
'map': {
'foo': {'desc': 'thing1', 'example': 'bar'},
'baz': {'desc': 'thing2', 'example': 'quux'},
}
}
module.remove_examples(schema)
assert schema == {'map': {'foo': {'desc': 'thing1'}, 'baz': {'desc': 'thing2'}}}
def test_remove_examples_strips_examples_from_sequence_of_maps():
schema = {'seq': [{'map': {'foo': {'desc': 'thing', 'example': 'bar'}}, 'example': 'stuff'}]}
module.remove_examples(schema)
assert schema == {'seq': [{'map': {'foo': {'desc': 'thing'}}}]}
def test_normalize_repository_path_passes_through_remote_repository():
repository = 'example.org:test.borg'