Database dump hooks for PostgreSQL, so you can easily dump your databases before backups run (#225).
This commit is contained in:
parent
fa5fa1c11b
commit
458e7776c5
12 changed files with 366 additions and 33 deletions
10
README.md
10
README.md
|
@ -41,10 +41,18 @@ retention:
|
|||
keep_monthly: 6
|
||||
|
||||
consistency:
|
||||
# List of consistency checks to run: "repository", "archives", or both.
|
||||
# List of consistency checks to run: "repository", "archives", etc.
|
||||
checks:
|
||||
- repository
|
||||
- archives
|
||||
|
||||
hooks:
|
||||
# Preparation scripts to run, databases to dump, and monitoring to perform.
|
||||
before_backup:
|
||||
- prepare-for-backup.sh
|
||||
postgresql_databases:
|
||||
- name: users
|
||||
healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
|
||||
```
|
||||
|
||||
borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
|
||||
|
|
|
@ -94,6 +94,20 @@ def _make_exclude_flags(location_config, exclude_filename=None):
|
|||
return exclude_from_flags + caches_flag + if_present_flags
|
||||
|
||||
|
||||
BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'
|
||||
|
||||
|
||||
def borgmatic_source_directories():
|
||||
'''
|
||||
Return a list of borgmatic-specific source directories used for state like database backups.
|
||||
'''
|
||||
return (
|
||||
[BORGMATIC_SOURCE_DIRECTORY]
|
||||
if os.path.exists(os.path.expanduser(BORGMATIC_SOURCE_DIRECTORY))
|
||||
else []
|
||||
)
|
||||
|
||||
|
||||
def create_archive(
|
||||
dry_run,
|
||||
repository,
|
||||
|
@ -109,7 +123,9 @@ def create_archive(
|
|||
Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
|
||||
storage config dict, create a Borg archive and return Borg's JSON output (if any).
|
||||
'''
|
||||
sources = _expand_directories(location_config['source_directories'])
|
||||
sources = _expand_directories(
|
||||
location_config['source_directories'] + borgmatic_source_directories()
|
||||
)
|
||||
|
||||
pattern_file = _write_pattern_file(location_config.get('patterns'))
|
||||
exclude_file = _write_pattern_file(
|
||||
|
|
|
@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list
|
|||
from borgmatic.borg import prune as borg_prune
|
||||
from borgmatic.commands.arguments import parse_arguments
|
||||
from borgmatic.config import checks, collect, convert, validate
|
||||
from borgmatic.hooks import command, healthchecks
|
||||
from borgmatic.hooks import command, healthchecks, postgresql
|
||||
from borgmatic.logger import configure_logging, should_do_markup
|
||||
from borgmatic.signals import configure_signals
|
||||
from borgmatic.verbosity import verbosity_to_log_level
|
||||
|
@ -60,6 +60,9 @@ def run_configuration(config_filename, config, arguments):
|
|||
'pre-backup',
|
||||
global_arguments.dry_run,
|
||||
)
|
||||
postgresql.dump_databases(
|
||||
hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run
|
||||
)
|
||||
healthchecks.ping_healthchecks(
|
||||
hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start'
|
||||
)
|
||||
|
@ -98,6 +101,9 @@ def run_configuration(config_filename, config, arguments):
|
|||
'post-backup',
|
||||
global_arguments.dry_run,
|
||||
)
|
||||
postgresql.remove_database_dumps(
|
||||
hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run
|
||||
)
|
||||
healthchecks.ping_healthchecks(
|
||||
hooks.get('healthchecks'), config_filename, global_arguments.dry_run
|
||||
)
|
||||
|
|
|
@ -367,6 +367,62 @@ map:
|
|||
occurs during a backup or when running a before_backup or after_backup hook.
|
||||
example:
|
||||
- echo "Error while creating a backup or running a backup hook."
|
||||
postgresql_databases:
|
||||
seq:
|
||||
- map:
|
||||
name:
|
||||
required: true
|
||||
type: str
|
||||
desc: |
|
||||
Database name (required if using this hook). Or "all" to dump all
|
||||
databases on the host.
|
||||
example: users
|
||||
hostname:
|
||||
type: str
|
||||
desc: |
|
||||
Database hostname to connect to. Defaults to connecting via local
|
||||
Unix socket.
|
||||
example: database.example.org
|
||||
port:
|
||||
type: int
|
||||
desc: Port to connect to. Defaults to 5432.
|
||||
example: 5433
|
||||
username:
|
||||
type: str
|
||||
desc: |
|
||||
Username with which to connect to the database. Defaults to the
|
||||
username of the current user. You probably want to specify the
|
||||
"postgres" superuser here when the database name is "all".
|
||||
example: dbuser
|
||||
password:
|
||||
type: str
|
||||
desc: |
|
||||
Password with which to connect to the database. Omitting a password
|
||||
will only work if PostgreSQL is configured to trust the configured
|
||||
username without a password, or you create a ~/.pgpass file.
|
||||
example: trustsome1
|
||||
format:
|
||||
type: str
|
||||
enum: ['plain', 'custom', 'directory', 'tar']
|
||||
desc: |
|
||||
Database dump output format. One of "plain", "custom", "directory",
|
||||
or "tar". Defaults to "custom" (unlike raw pg_dump). See
|
||||
https://www.postgresql.org/docs/current/app-pgdump.html for details.
|
||||
Note that format is ignored when the database name is "all".
|
||||
example: directory
|
||||
options:
|
||||
type: str
|
||||
desc: |
|
||||
Additional pg_dump/pg_dumpall options to pass directly to the dump
|
||||
command, without performing any validation on them. See
|
||||
https://www.postgresql.org/docs/current/app-pgdump.html for details.
|
||||
example: --role=someone
|
||||
desc: |
|
||||
List of one or more PostgreSQL databases to dump before creating a backup,
|
||||
run once per configuration file. The database dumps are added to your source
|
||||
directories at runtime, backed up, and then removed afterwards. Requires
|
||||
pg_dump/pg_dumpall/pg_restore commands. See
|
||||
https://www.postgresql.org/docs/current/app-pgdump.html for details.
|
||||
healthchecks:
|
||||
type: str
|
||||
desc: |
|
||||
|
|
|
@ -64,6 +64,23 @@ def apply_logical_validation(config_filename, parsed_configuration):
|
|||
)
|
||||
|
||||
|
||||
def remove_examples(schema):
|
||||
'''
|
||||
pykwalify gets angry if the example field is not a string. So rather than bend to its will,
|
||||
remove all examples from the given schema before passing the schema to pykwalify.
|
||||
'''
|
||||
if 'map' in schema:
|
||||
for item_name, item_schema in schema['map'].items():
|
||||
item_schema.pop('example', None)
|
||||
remove_examples(item_schema)
|
||||
elif 'seq' in schema:
|
||||
for item_schema in schema['seq']:
|
||||
item_schema.pop('example', None)
|
||||
remove_examples(item_schema)
|
||||
|
||||
return schema
|
||||
|
||||
|
||||
def parse_configuration(config_filename, schema_filename):
|
||||
'''
|
||||
Given the path to a config filename in YAML format and the path to a schema filename in
|
||||
|
@ -84,13 +101,7 @@ def parse_configuration(config_filename, schema_filename):
|
|||
except (ruamel.yaml.error.YAMLError, RecursionError) as error:
|
||||
raise Validation_error(config_filename, (str(error),))
|
||||
|
||||
# pykwalify gets angry if the example field is not a string. So rather than bend to its will,
|
||||
# remove all examples before passing the schema to pykwalify.
|
||||
for section_name, section_schema in schema['map'].items():
|
||||
for field_name, field_schema in section_schema['map'].items():
|
||||
field_schema.pop('example', None)
|
||||
|
||||
validator = pykwalify.core.Core(source_data=config, schema_data=schema)
|
||||
validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
|
||||
parsed_result = validator.validate(raise_exception=False)
|
||||
|
||||
if validator.validation_errors:
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
@ -8,10 +9,17 @@ ERROR_OUTPUT_MAX_LINE_COUNT = 25
|
|||
BORG_ERROR_EXIT_CODE = 2
|
||||
|
||||
|
||||
def execute_and_log_output(full_command, output_log_level, shell):
|
||||
def borg_command(full_command):
|
||||
'''
|
||||
Return True if this is a Borg command, or False if it's some other command.
|
||||
'''
|
||||
return 'borg' in full_command[0]
|
||||
|
||||
|
||||
def execute_and_log_output(full_command, output_log_level, shell, environment):
|
||||
last_lines = []
|
||||
process = subprocess.Popen(
|
||||
full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell
|
||||
full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, env=environment
|
||||
)
|
||||
|
||||
while process.poll() is None:
|
||||
|
@ -33,9 +41,11 @@ def execute_and_log_output(full_command, output_log_level, shell):
|
|||
|
||||
exit_code = process.poll()
|
||||
|
||||
# If shell is True, assume we're running something other than Borg and should treat all non-zero
|
||||
# exit codes as errors.
|
||||
error = bool(exit_code != 0) if shell else bool(exit_code >= BORG_ERROR_EXIT_CODE)
|
||||
# If we're running something other than Borg, treat all non-zero exit codes as errors.
|
||||
if borg_command(full_command):
|
||||
error = bool(exit_code >= BORG_ERROR_EXIT_CODE)
|
||||
else:
|
||||
error = bool(exit_code != 0)
|
||||
|
||||
if error:
|
||||
# If an error occurs, include its output in the raised exception so that we don't
|
||||
|
@ -48,21 +58,25 @@ def execute_and_log_output(full_command, output_log_level, shell):
|
|||
)
|
||||
|
||||
|
||||
def execute_command(full_command, output_log_level=logging.INFO, shell=False):
|
||||
def execute_command(
|
||||
full_command, output_log_level=logging.INFO, shell=False, extra_environment=None
|
||||
):
|
||||
'''
|
||||
Execute the given command (a sequence of command/argument strings) and log its output at the
|
||||
given log level. If output log level is None, instead capture and return the output. If
|
||||
shell is True, execute the command within a shell.
|
||||
shell is True, execute the command within a shell. If an extra environment dict is given, then
|
||||
use it to augment the current environment, and pass the result into the command.
|
||||
|
||||
Raise subprocesses.CalledProcessError if an error occurs while running the command.
|
||||
'''
|
||||
logger.debug(' '.join(full_command))
|
||||
environment = {**os.environ, **extra_environment} if extra_environment else None
|
||||
|
||||
if output_log_level is None:
|
||||
output = subprocess.check_output(full_command, shell=shell)
|
||||
output = subprocess.check_output(full_command, shell=shell, env=environment)
|
||||
return output.decode() if output is not None else None
|
||||
else:
|
||||
execute_and_log_output(full_command, output_log_level, shell=shell)
|
||||
execute_and_log_output(full_command, output_log_level, shell=shell, environment=environment)
|
||||
|
||||
|
||||
def execute_command_without_capture(full_command):
|
||||
|
|
88
borgmatic/hooks/postgresql.py
Normal file
88
borgmatic/hooks/postgresql.py
Normal file
|
@ -0,0 +1,88 @@
|
|||
import logging
|
||||
import os
|
||||
|
||||
from borgmatic.execute import execute_command
|
||||
|
||||
DUMP_PATH = '~/.borgmatic/postgresql_databases'
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def dump_databases(databases, config_filename, dry_run):
|
||||
'''
|
||||
Dump the given PostgreSQL databases to disk. The databases are supplied as a sequence of dicts,
|
||||
one dict describing each database as per the configuration schema. Use the given configuration
|
||||
filename in any log entries. If this is a dry run, then don't actually dump anything.
|
||||
'''
|
||||
if not databases:
|
||||
logger.debug('{}: No PostgreSQL databases configured'.format(config_filename))
|
||||
return
|
||||
|
||||
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
|
||||
|
||||
logger.info('{}: Dumping PostgreSQL databases{}'.format(config_filename, dry_run_label))
|
||||
|
||||
for database in databases:
|
||||
if os.path.sep in database['name']:
|
||||
raise ValueError('Invalid database name {}'.format(database['name']))
|
||||
|
||||
dump_path = os.path.join(
|
||||
os.path.expanduser(DUMP_PATH), database.get('hostname', 'localhost')
|
||||
)
|
||||
name = database['name']
|
||||
all_databases = bool(name == 'all')
|
||||
command = (
|
||||
('pg_dumpall' if all_databases else 'pg_dump', '--no-password', '--clean')
|
||||
+ ('--file', os.path.join(dump_path, name))
|
||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||
+ (('--username', database['username']) if 'username' in database else ())
|
||||
+ (() if all_databases else ('--format', database.get('format', 'custom')))
|
||||
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
||||
+ (() if all_databases else (name,))
|
||||
)
|
||||
extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
|
||||
|
||||
logger.debug(
|
||||
'{}: Dumping PostgreSQL database {}{}'.format(config_filename, name, dry_run_label)
|
||||
)
|
||||
if not dry_run:
|
||||
os.makedirs(dump_path, mode=0o700, exist_ok=True)
|
||||
execute_command(command, extra_environment=extra_environment)
|
||||
|
||||
|
||||
def remove_database_dumps(databases, config_filename, dry_run):
|
||||
'''
|
||||
Remove the database dumps for the given databases. The databases are supplied as a sequence of
|
||||
dicts, one dict describing each database as per the configuration schema. Use the given
|
||||
configuration filename in any log entries. If this is a dry run, then don't actually remove
|
||||
anything.
|
||||
'''
|
||||
if not databases:
|
||||
logger.debug('{}: No PostgreSQL databases configured'.format(config_filename))
|
||||
return
|
||||
|
||||
dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
|
||||
|
||||
logger.info('{}: Removing PostgreSQL database dumps{}'.format(config_filename, dry_run_label))
|
||||
|
||||
for database in databases:
|
||||
if os.path.sep in database['name']:
|
||||
raise ValueError('Invalid database name {}'.format(database['name']))
|
||||
|
||||
name = database['name']
|
||||
dump_path = os.path.join(
|
||||
os.path.expanduser(DUMP_PATH), database.get('hostname', 'localhost')
|
||||
)
|
||||
dump_filename = os.path.join(dump_path, name)
|
||||
|
||||
logger.debug(
|
||||
'{}: Remove PostgreSQL database dump {} from {}{}'.format(
|
||||
config_filename, name, dump_filename, dry_run_label
|
||||
)
|
||||
)
|
||||
if dry_run:
|
||||
continue
|
||||
|
||||
os.remove(dump_filename)
|
||||
if len(os.listdir(dump_path)) == 0:
|
||||
os.rmdir(dump_path)
|
|
@ -7,36 +7,57 @@ from flexmock import flexmock
|
|||
from borgmatic import execute as module
|
||||
|
||||
|
||||
def test_borg_command_identifies_borg_command():
|
||||
assert module.borg_command(['/usr/bin/borg1', 'info'])
|
||||
|
||||
|
||||
def test_borg_command_does_not_identify_other_command():
|
||||
assert not module.borg_command(['grep', 'stuff'])
|
||||
|
||||
|
||||
def test_execute_and_log_output_logs_each_line_separately():
|
||||
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').once()
|
||||
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
|
||||
flexmock(module).should_receive('borg_command').and_return(False)
|
||||
|
||||
module.execute_and_log_output(['echo', 'hi'], output_log_level=logging.INFO, shell=False)
|
||||
module.execute_and_log_output(['echo', 'there'], output_log_level=logging.INFO, shell=False)
|
||||
module.execute_and_log_output(
|
||||
['echo', 'hi'], output_log_level=logging.INFO, shell=False, environment=None
|
||||
)
|
||||
module.execute_and_log_output(
|
||||
['echo', 'there'], output_log_level=logging.INFO, shell=False, environment=None
|
||||
)
|
||||
|
||||
|
||||
def test_execute_and_log_output_with_borg_warning_does_not_raise():
|
||||
flexmock(module.logger).should_receive('log')
|
||||
flexmock(module).should_receive('borg_command').and_return(True)
|
||||
|
||||
# Borg's exit code 1 is a warning, not an error.
|
||||
module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=False)
|
||||
module.execute_and_log_output(
|
||||
['false'], output_log_level=logging.INFO, shell=False, environment=None
|
||||
)
|
||||
|
||||
|
||||
def test_execute_and_log_output_includes_borg_error_output_in_exception():
|
||||
flexmock(module.logger).should_receive('log')
|
||||
flexmock(module).should_receive('borg_command').and_return(True)
|
||||
|
||||
with pytest.raises(subprocess.CalledProcessError) as error:
|
||||
module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
|
||||
module.execute_and_log_output(
|
||||
['grep'], output_log_level=logging.INFO, shell=False, environment=None
|
||||
)
|
||||
|
||||
assert error.value.returncode == 2
|
||||
assert error.value.output
|
||||
|
||||
|
||||
def test_execute_and_log_output_with_shell_error_raises():
|
||||
def test_execute_and_log_output_with_non_borg_error_raises():
|
||||
flexmock(module.logger).should_receive('log')
|
||||
flexmock(module).should_receive('borg_command').and_return(False)
|
||||
|
||||
with pytest.raises(subprocess.CalledProcessError) as error:
|
||||
module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=True)
|
||||
module.execute_and_log_output(
|
||||
['false'], output_log_level=logging.INFO, shell=False, environment=None
|
||||
)
|
||||
|
||||
assert error.value.returncode == 1
|
||||
|
||||
|
@ -44,9 +65,12 @@ def test_execute_and_log_output_with_shell_error_raises():
|
|||
def test_execute_and_log_output_truncates_long_borg_error_output():
|
||||
flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
|
||||
flexmock(module.logger).should_receive('log')
|
||||
flexmock(module).should_receive('borg_command').and_return(False)
|
||||
|
||||
with pytest.raises(subprocess.CalledProcessError) as error:
|
||||
module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
|
||||
module.execute_and_log_output(
|
||||
['grep'], output_log_level=logging.INFO, shell=False, environment=None
|
||||
)
|
||||
|
||||
assert error.value.returncode == 2
|
||||
assert error.value.output.startswith('...')
|
||||
|
@ -54,12 +78,18 @@ def test_execute_and_log_output_truncates_long_borg_error_output():
|
|||
|
||||
def test_execute_and_log_output_with_no_output_logs_nothing():
|
||||
flexmock(module.logger).should_receive('log').never()
|
||||
flexmock(module).should_receive('borg_command').and_return(False)
|
||||
|
||||
module.execute_and_log_output(['true'], output_log_level=logging.INFO, shell=False)
|
||||
module.execute_and_log_output(
|
||||
['true'], output_log_level=logging.INFO, shell=False, environment=None
|
||||
)
|
||||
|
||||
|
||||
def test_execute_and_log_output_with_error_exit_status_raises():
|
||||
flexmock(module.logger).should_receive('log')
|
||||
flexmock(module).should_receive('borg_command').and_return(False)
|
||||
|
||||
with pytest.raises(subprocess.CalledProcessError):
|
||||
module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
|
||||
module.execute_and_log_output(
|
||||
['grep'], output_log_level=logging.INFO, shell=False, environment=None
|
||||
)
|
||||
|
|
|
@ -156,11 +156,26 @@ def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
|
|||
assert exclude_flags == ()
|
||||
|
||||
|
||||
def test_borgmatic_source_directories_set_when_directory_exists():
|
||||
flexmock(module.os.path).should_receive('exists').and_return(True)
|
||||
flexmock(module.os.path).should_receive('expanduser')
|
||||
|
||||
assert module.borgmatic_source_directories() == [module.BORGMATIC_SOURCE_DIRECTORY]
|
||||
|
||||
|
||||
def test_borgmatic_source_directories_empty_when_directory_does_not_exist():
|
||||
flexmock(module.os.path).should_receive('exists').and_return(False)
|
||||
flexmock(module.os.path).should_receive('expanduser')
|
||||
|
||||
assert module.borgmatic_source_directories() == []
|
||||
|
||||
|
||||
DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
|
||||
ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
|
||||
|
||||
|
||||
def test_create_archive_calls_borg_with_parameters():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -184,6 +199,7 @@ def test_create_archive_calls_borg_with_parameters():
|
|||
|
||||
def test_create_archive_with_patterns_calls_borg_with_patterns():
|
||||
pattern_flags = ('--patterns-from', 'patterns')
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(
|
||||
|
@ -209,6 +225,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
|
|||
|
||||
def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
|
||||
exclude_flags = ('--exclude-from', 'excludes')
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
|
||||
|
@ -233,6 +250,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
|
|||
|
||||
|
||||
def test_create_archive_with_log_info_calls_borg_with_info_parameter():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -258,6 +276,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
|
|||
|
||||
|
||||
def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -283,6 +302,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
|
|||
|
||||
|
||||
def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -308,6 +328,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
|
|||
|
||||
|
||||
def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -332,6 +353,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
|
|||
|
||||
|
||||
def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -357,6 +379,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
|
|||
def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_parameter():
|
||||
# --dry-run and --stats are mutually exclusive, see:
|
||||
# https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -385,6 +408,7 @@ def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_param
|
|||
def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_parameter():
|
||||
# --dry-run and --stats are mutually exclusive, see:
|
||||
# https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -411,6 +435,7 @@ def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_para
|
|||
|
||||
|
||||
def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -434,6 +459,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
|
|||
|
||||
|
||||
def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -457,6 +483,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
|
|||
|
||||
|
||||
def test_create_archive_with_compression_calls_borg_with_compression_parameters():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -480,6 +507,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
|
|||
|
||||
|
||||
def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -503,6 +531,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
|
|||
|
||||
|
||||
def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -526,6 +555,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
|
|||
|
||||
|
||||
def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -549,6 +579,7 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
|
|||
|
||||
|
||||
def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -573,6 +604,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
|
|||
|
||||
@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
|
||||
def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -597,6 +629,7 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
|
|||
|
||||
@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
|
||||
def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -621,6 +654,7 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
|
|||
|
||||
|
||||
def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -645,6 +679,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
|
|||
|
||||
|
||||
def test_create_archive_with_local_path_calls_borg_via_local_path():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -668,6 +703,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
|
|||
|
||||
|
||||
def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -692,6 +728,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
|
|||
|
||||
|
||||
def test_create_archive_with_umask_calls_borg_with_umask_parameters():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -714,6 +751,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
|
|||
|
||||
|
||||
def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -736,6 +774,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
|
|||
|
||||
|
||||
def test_create_archive_with_stats_calls_borg_with_stats_parameter():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -759,6 +798,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter():
|
|||
|
||||
|
||||
def test_create_archive_with_progress_calls_borg_with_progress_parameter():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -782,6 +822,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
|
|||
|
||||
|
||||
def test_create_archive_with_json_calls_borg_with_json_parameter():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -807,6 +848,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
|
|||
|
||||
|
||||
def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -833,6 +875,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
|
|||
|
||||
|
||||
def test_create_archive_with_source_directories_glob_expands():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -857,6 +900,7 @@ def test_create_archive_with_source_directories_glob_expands():
|
|||
|
||||
|
||||
def test_create_archive_with_non_matching_source_directories_glob_passes_through():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo*',))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -881,6 +925,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
|
|||
|
||||
|
||||
def test_create_archive_with_glob_calls_borg_with_expanded_directories():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -904,6 +949,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
|
|||
|
||||
|
||||
def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
@ -926,6 +972,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
|
|||
|
||||
|
||||
def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
|
||||
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
|
||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
|
||||
flexmock(module).should_receive('_expand_home_directories').and_return(())
|
||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||
|
|
|
@ -23,6 +23,9 @@ def test_run_configuration_runs_actions_for_each_repository():
|
|||
def test_run_configuration_executes_hooks_for_create_action():
|
||||
flexmock(module.borg_environment).should_receive('initialize')
|
||||
flexmock(module.command).should_receive('execute_hook').twice()
|
||||
flexmock(module.postgresql).should_receive('dump_databases').once()
|
||||
flexmock(module.healthchecks).should_receive('ping_healthchecks').twice()
|
||||
flexmock(module.postgresql).should_receive('remove_database_dumps').once()
|
||||
flexmock(module).should_receive('run_actions').and_return([])
|
||||
config = {'location': {'repositories': ['foo']}}
|
||||
arguments = {'global': flexmock(dry_run=False), 'create': flexmock()}
|
||||
|
@ -33,6 +36,8 @@ def test_run_configuration_executes_hooks_for_create_action():
|
|||
def test_run_configuration_logs_actions_error():
|
||||
flexmock(module.borg_environment).should_receive('initialize')
|
||||
flexmock(module.command).should_receive('execute_hook')
|
||||
flexmock(module.postgresql).should_receive('dump_databases')
|
||||
flexmock(module.healthchecks).should_receive('ping_healthchecks')
|
||||
expected_results = [flexmock()]
|
||||
flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
|
||||
flexmock(module).should_receive('run_actions').and_raise(OSError)
|
||||
|
|
|
@ -74,6 +74,27 @@ def test_apply_logical_validation_does_not_raise_otherwise():
|
|||
module.apply_logical_validation('config.yaml', {'retention': {'keep_secondly': 1000}})
|
||||
|
||||
|
||||
def test_remove_examples_strips_examples_from_map():
|
||||
schema = {
|
||||
'map': {
|
||||
'foo': {'desc': 'thing1', 'example': 'bar'},
|
||||
'baz': {'desc': 'thing2', 'example': 'quux'},
|
||||
}
|
||||
}
|
||||
|
||||
module.remove_examples(schema)
|
||||
|
||||
assert schema == {'map': {'foo': {'desc': 'thing1'}, 'baz': {'desc': 'thing2'}}}
|
||||
|
||||
|
||||
def test_remove_examples_strips_examples_from_sequence_of_maps():
|
||||
schema = {'seq': [{'map': {'foo': {'desc': 'thing', 'example': 'bar'}}, 'example': 'stuff'}]}
|
||||
|
||||
module.remove_examples(schema)
|
||||
|
||||
assert schema == {'seq': [{'map': {'foo': {'desc': 'thing'}}}]}
|
||||
|
||||
|
||||
def test_guard_configuration_contains_repository_does_not_raise_when_repository_in_config():
|
||||
module.guard_configuration_contains_repository(
|
||||
repository='repo', configurations={'config.yaml': {'location': {'repositories': ['repo']}}}
|
||||
|
|
|
@ -8,8 +8,9 @@ from borgmatic import execute as module
|
|||
|
||||
def test_execute_command_calls_full_command():
|
||||
full_command = ['foo', 'bar']
|
||||
flexmock(module.os, environ={'a': 'b'})
|
||||
flexmock(module).should_receive('execute_and_log_output').with_args(
|
||||
full_command, output_log_level=logging.INFO, shell=False
|
||||
full_command, output_log_level=logging.INFO, shell=False, environment=None
|
||||
).once()
|
||||
|
||||
output = module.execute_command(full_command)
|
||||
|
@ -19,8 +20,9 @@ def test_execute_command_calls_full_command():
|
|||
|
||||
def test_execute_command_calls_full_command_with_shell():
|
||||
full_command = ['foo', 'bar']
|
||||
flexmock(module.os, environ={'a': 'b'})
|
||||
flexmock(module).should_receive('execute_and_log_output').with_args(
|
||||
full_command, output_log_level=logging.INFO, shell=True
|
||||
full_command, output_log_level=logging.INFO, shell=True, environment=None
|
||||
).once()
|
||||
|
||||
output = module.execute_command(full_command, shell=True)
|
||||
|
@ -28,11 +30,24 @@ def test_execute_command_calls_full_command_with_shell():
|
|||
assert output is None
|
||||
|
||||
|
||||
def test_execute_command_calls_full_command_with_extra_environment():
|
||||
full_command = ['foo', 'bar']
|
||||
flexmock(module.os, environ={'a': 'b'})
|
||||
flexmock(module).should_receive('execute_and_log_output').with_args(
|
||||
full_command, output_log_level=logging.INFO, shell=False, environment={'a': 'b', 'c': 'd'}
|
||||
).once()
|
||||
|
||||
output = module.execute_command(full_command, extra_environment={'c': 'd'})
|
||||
|
||||
assert output is None
|
||||
|
||||
|
||||
def test_execute_command_captures_output():
|
||||
full_command = ['foo', 'bar']
|
||||
expected_output = '[]'
|
||||
flexmock(module.os, environ={'a': 'b'})
|
||||
flexmock(module.subprocess).should_receive('check_output').with_args(
|
||||
full_command, shell=False
|
||||
full_command, shell=False, env=None
|
||||
).and_return(flexmock(decode=lambda: expected_output)).once()
|
||||
|
||||
output = module.execute_command(full_command, output_log_level=None)
|
||||
|
@ -43,8 +58,9 @@ def test_execute_command_captures_output():
|
|||
def test_execute_command_captures_output_with_shell():
|
||||
full_command = ['foo', 'bar']
|
||||
expected_output = '[]'
|
||||
flexmock(module.os, environ={'a': 'b'})
|
||||
flexmock(module.subprocess).should_receive('check_output').with_args(
|
||||
full_command, shell=True
|
||||
full_command, shell=True, env=None
|
||||
).and_return(flexmock(decode=lambda: expected_output)).once()
|
||||
|
||||
output = module.execute_command(full_command, output_log_level=None, shell=True)
|
||||
|
@ -52,6 +68,21 @@ def test_execute_command_captures_output_with_shell():
|
|||
assert output == expected_output
|
||||
|
||||
|
||||
def test_execute_command_captures_output_with_extra_environment():
|
||||
full_command = ['foo', 'bar']
|
||||
expected_output = '[]'
|
||||
flexmock(module.os, environ={'a': 'b'})
|
||||
flexmock(module.subprocess).should_receive('check_output').with_args(
|
||||
full_command, shell=False, env={'a': 'b', 'c': 'd'}
|
||||
).and_return(flexmock(decode=lambda: expected_output)).once()
|
||||
|
||||
output = module.execute_command(
|
||||
full_command, output_log_level=None, shell=False, extra_environment={'c': 'd'}
|
||||
)
|
||||
|
||||
assert output == expected_output
|
||||
|
||||
|
||||
def test_execute_command_without_capture_does_not_raise_on_success():
|
||||
flexmock(module.subprocess).should_receive('check_call').and_raise(
|
||||
module.subprocess.CalledProcessError(0, 'borg init')
|
||||
|
|
Loading…
Reference in a new issue