Database dump hooks for PostgreSQL, so you can easily dump your databases before backups run (#225).

This commit is contained in:
Dan Helfman 2019-10-22 16:28:42 -07:00
parent fa5fa1c11b
commit 458e7776c5
12 changed files with 366 additions and 33 deletions

View file

@ -41,10 +41,18 @@ retention:
keep_monthly: 6
consistency:
# List of consistency checks to run: "repository", "archives", or both.
# List of consistency checks to run: "repository", "archives", etc.
checks:
- repository
- archives
hooks:
# Preparation scripts to run, databases to dump, and monitoring to perform.
before_backup:
- prepare-for-backup.sh
postgresql_databases:
- name: users
healthchecks: https://hc-ping.com/be067061-cf96-4412-8eae-62b0c50d6a8c
```
borgmatic is hosted at <https://torsion.org/borgmatic> with [source code

View file

@ -94,6 +94,20 @@ def _make_exclude_flags(location_config, exclude_filename=None):
return exclude_from_flags + caches_flag + if_present_flags
BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'
def borgmatic_source_directories():
'''
Return a list of borgmatic-specific source directories used for state like database backups.
'''
return (
[BORGMATIC_SOURCE_DIRECTORY]
if os.path.exists(os.path.expanduser(BORGMATIC_SOURCE_DIRECTORY))
else []
)
def create_archive(
dry_run,
repository,
@ -109,7 +123,9 @@ def create_archive(
Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
storage config dict, create a Borg archive and return Borg's JSON output (if any).
'''
sources = _expand_directories(location_config['source_directories'])
sources = _expand_directories(
location_config['source_directories'] + borgmatic_source_directories()
)
pattern_file = _write_pattern_file(location_config.get('patterns'))
exclude_file = _write_pattern_file(

View file

@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list
from borgmatic.borg import prune as borg_prune
from borgmatic.commands.arguments import parse_arguments
from borgmatic.config import checks, collect, convert, validate
from borgmatic.hooks import command, healthchecks
from borgmatic.hooks import command, healthchecks, postgresql
from borgmatic.logger import configure_logging, should_do_markup
from borgmatic.signals import configure_signals
from borgmatic.verbosity import verbosity_to_log_level
@ -60,6 +60,9 @@ def run_configuration(config_filename, config, arguments):
'pre-backup',
global_arguments.dry_run,
)
postgresql.dump_databases(
hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run
)
healthchecks.ping_healthchecks(
hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start'
)
@ -98,6 +101,9 @@ def run_configuration(config_filename, config, arguments):
'post-backup',
global_arguments.dry_run,
)
postgresql.remove_database_dumps(
hooks.get('postgresql_databases'), config_filename, global_arguments.dry_run
)
healthchecks.ping_healthchecks(
hooks.get('healthchecks'), config_filename, global_arguments.dry_run
)

View file

@ -367,6 +367,62 @@ map:
occurs during a backup or when running a before_backup or after_backup hook.
example:
- echo "Error while creating a backup or running a backup hook."
postgresql_databases:
seq:
- map:
name:
required: true
type: str
desc: |
Database name (required if using this hook). Or "all" to dump all
databases on the host.
example: users
hostname:
type: str
desc: |
Database hostname to connect to. Defaults to connecting via local
Unix socket.
example: database.example.org
port:
type: int
desc: Port to connect to. Defaults to 5432.
example: 5433
username:
type: str
desc: |
Username with which to connect to the database. Defaults to the
username of the current user. You probably want to specify the
"postgres" superuser here when the database name is "all".
example: dbuser
password:
type: str
desc: |
Password with which to connect to the database. Omitting a password
will only work if PostgreSQL is configured to trust the configured
username without a password, or you create a ~/.pgpass file.
example: trustsome1
format:
type: str
enum: ['plain', 'custom', 'directory', 'tar']
desc: |
Database dump output format. One of "plain", "custom", "directory",
or "tar". Defaults to "custom" (unlike raw pg_dump). See
https://www.postgresql.org/docs/current/app-pgdump.html for details.
Note that format is ignored when the database name is "all".
example: directory
options:
type: str
desc: |
Additional pg_dump/pg_dumpall options to pass directly to the dump
command, without performing any validation on them. See
https://www.postgresql.org/docs/current/app-pgdump.html for details.
example: --role=someone
desc: |
List of one or more PostgreSQL databases to dump before creating a backup,
run once per configuration file. The database dumps are added to your source
directories at runtime, backed up, and then removed afterwards. Requires
pg_dump/pg_dumpall/pg_restore commands. See
https://www.postgresql.org/docs/current/app-pgdump.html for details.
healthchecks:
type: str
desc: |

View file

@ -64,6 +64,23 @@ def apply_logical_validation(config_filename, parsed_configuration):
)
def remove_examples(schema):
'''
pykwalify gets angry if the example field is not a string. So rather than bend to its will,
remove all examples from the given schema before passing the schema to pykwalify.
'''
if 'map' in schema:
for item_name, item_schema in schema['map'].items():
item_schema.pop('example', None)
remove_examples(item_schema)
elif 'seq' in schema:
for item_schema in schema['seq']:
item_schema.pop('example', None)
remove_examples(item_schema)
return schema
def parse_configuration(config_filename, schema_filename):
'''
Given the path to a config filename in YAML format and the path to a schema filename in
@ -84,13 +101,7 @@ def parse_configuration(config_filename, schema_filename):
except (ruamel.yaml.error.YAMLError, RecursionError) as error:
raise Validation_error(config_filename, (str(error),))
# pykwalify gets angry if the example field is not a string. So rather than bend to its will,
# remove all examples before passing the schema to pykwalify.
for section_name, section_schema in schema['map'].items():
for field_name, field_schema in section_schema['map'].items():
field_schema.pop('example', None)
validator = pykwalify.core.Core(source_data=config, schema_data=schema)
validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
parsed_result = validator.validate(raise_exception=False)
if validator.validation_errors:

View file

@ -1,4 +1,5 @@
import logging
import os
import subprocess
logger = logging.getLogger(__name__)
@ -8,10 +9,17 @@ ERROR_OUTPUT_MAX_LINE_COUNT = 25
BORG_ERROR_EXIT_CODE = 2
def execute_and_log_output(full_command, output_log_level, shell):
def borg_command(full_command):
'''
Return True if this is a Borg command, or False if it's some other command.
'''
return 'borg' in full_command[0]
def execute_and_log_output(full_command, output_log_level, shell, environment):
last_lines = []
process = subprocess.Popen(
full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell
full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell, env=environment
)
while process.poll() is None:
@ -33,9 +41,11 @@ def execute_and_log_output(full_command, output_log_level, shell):
exit_code = process.poll()
# If shell is True, assume we're running something other than Borg and should treat all non-zero
# exit codes as errors.
error = bool(exit_code != 0) if shell else bool(exit_code >= BORG_ERROR_EXIT_CODE)
# If we're running something other than Borg, treat all non-zero exit codes as errors.
if borg_command(full_command):
error = bool(exit_code >= BORG_ERROR_EXIT_CODE)
else:
error = bool(exit_code != 0)
if error:
# If an error occurs, include its output in the raised exception so that we don't
@ -48,21 +58,25 @@ def execute_and_log_output(full_command, output_log_level, shell):
)
def execute_command(full_command, output_log_level=logging.INFO, shell=False):
def execute_command(
full_command, output_log_level=logging.INFO, shell=False, extra_environment=None
):
'''
Execute the given command (a sequence of command/argument strings) and log its output at the
given log level. If output log level is None, instead capture and return the output. If
shell is True, execute the command within a shell.
shell is True, execute the command within a shell. If an extra environment dict is given, then
use it to augment the current environment, and pass the result into the command.
Raise subprocesses.CalledProcessError if an error occurs while running the command.
'''
logger.debug(' '.join(full_command))
environment = {**os.environ, **extra_environment} if extra_environment else None
if output_log_level is None:
output = subprocess.check_output(full_command, shell=shell)
output = subprocess.check_output(full_command, shell=shell, env=environment)
return output.decode() if output is not None else None
else:
execute_and_log_output(full_command, output_log_level, shell=shell)
execute_and_log_output(full_command, output_log_level, shell=shell, environment=environment)
def execute_command_without_capture(full_command):

View file

@ -0,0 +1,88 @@
import logging
import os
from borgmatic.execute import execute_command
DUMP_PATH = '~/.borgmatic/postgresql_databases'
logger = logging.getLogger(__name__)
def dump_databases(databases, config_filename, dry_run):
'''
Dump the given PostgreSQL databases to disk. The databases are supplied as a sequence of dicts,
one dict describing each database as per the configuration schema. Use the given configuration
filename in any log entries. If this is a dry run, then don't actually dump anything.
'''
if not databases:
logger.debug('{}: No PostgreSQL databases configured'.format(config_filename))
return
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
logger.info('{}: Dumping PostgreSQL databases{}'.format(config_filename, dry_run_label))
for database in databases:
if os.path.sep in database['name']:
raise ValueError('Invalid database name {}'.format(database['name']))
dump_path = os.path.join(
os.path.expanduser(DUMP_PATH), database.get('hostname', 'localhost')
)
name = database['name']
all_databases = bool(name == 'all')
command = (
('pg_dumpall' if all_databases else 'pg_dump', '--no-password', '--clean')
+ ('--file', os.path.join(dump_path, name))
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ())
+ (() if all_databases else ('--format', database.get('format', 'custom')))
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
+ (() if all_databases else (name,))
)
extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
logger.debug(
'{}: Dumping PostgreSQL database {}{}'.format(config_filename, name, dry_run_label)
)
if not dry_run:
os.makedirs(dump_path, mode=0o700, exist_ok=True)
execute_command(command, extra_environment=extra_environment)
def remove_database_dumps(databases, config_filename, dry_run):
'''
Remove the database dumps for the given databases. The databases are supplied as a sequence of
dicts, one dict describing each database as per the configuration schema. Use the given
configuration filename in any log entries. If this is a dry run, then don't actually remove
anything.
'''
if not databases:
logger.debug('{}: No PostgreSQL databases configured'.format(config_filename))
return
dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
logger.info('{}: Removing PostgreSQL database dumps{}'.format(config_filename, dry_run_label))
for database in databases:
if os.path.sep in database['name']:
raise ValueError('Invalid database name {}'.format(database['name']))
name = database['name']
dump_path = os.path.join(
os.path.expanduser(DUMP_PATH), database.get('hostname', 'localhost')
)
dump_filename = os.path.join(dump_path, name)
logger.debug(
'{}: Remove PostgreSQL database dump {} from {}{}'.format(
config_filename, name, dump_filename, dry_run_label
)
)
if dry_run:
continue
os.remove(dump_filename)
if len(os.listdir(dump_path)) == 0:
os.rmdir(dump_path)

View file

@ -7,36 +7,57 @@ from flexmock import flexmock
from borgmatic import execute as module
def test_borg_command_identifies_borg_command():
assert module.borg_command(['/usr/bin/borg1', 'info'])
def test_borg_command_does_not_identify_other_command():
assert not module.borg_command(['grep', 'stuff'])
def test_execute_and_log_output_logs_each_line_separately():
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').once()
flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
flexmock(module).should_receive('borg_command').and_return(False)
module.execute_and_log_output(['echo', 'hi'], output_log_level=logging.INFO, shell=False)
module.execute_and_log_output(['echo', 'there'], output_log_level=logging.INFO, shell=False)
module.execute_and_log_output(
['echo', 'hi'], output_log_level=logging.INFO, shell=False, environment=None
)
module.execute_and_log_output(
['echo', 'there'], output_log_level=logging.INFO, shell=False, environment=None
)
def test_execute_and_log_output_with_borg_warning_does_not_raise():
flexmock(module.logger).should_receive('log')
flexmock(module).should_receive('borg_command').and_return(True)
# Borg's exit code 1 is a warning, not an error.
module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=False)
module.execute_and_log_output(
['false'], output_log_level=logging.INFO, shell=False, environment=None
)
def test_execute_and_log_output_includes_borg_error_output_in_exception():
flexmock(module.logger).should_receive('log')
flexmock(module).should_receive('borg_command').and_return(True)
with pytest.raises(subprocess.CalledProcessError) as error:
module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
module.execute_and_log_output(
['grep'], output_log_level=logging.INFO, shell=False, environment=None
)
assert error.value.returncode == 2
assert error.value.output
def test_execute_and_log_output_with_shell_error_raises():
def test_execute_and_log_output_with_non_borg_error_raises():
flexmock(module.logger).should_receive('log')
flexmock(module).should_receive('borg_command').and_return(False)
with pytest.raises(subprocess.CalledProcessError) as error:
module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=True)
module.execute_and_log_output(
['false'], output_log_level=logging.INFO, shell=False, environment=None
)
assert error.value.returncode == 1
@ -44,9 +65,12 @@ def test_execute_and_log_output_with_shell_error_raises():
def test_execute_and_log_output_truncates_long_borg_error_output():
flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
flexmock(module.logger).should_receive('log')
flexmock(module).should_receive('borg_command').and_return(False)
with pytest.raises(subprocess.CalledProcessError) as error:
module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
module.execute_and_log_output(
['grep'], output_log_level=logging.INFO, shell=False, environment=None
)
assert error.value.returncode == 2
assert error.value.output.startswith('...')
@ -54,12 +78,18 @@ def test_execute_and_log_output_truncates_long_borg_error_output():
def test_execute_and_log_output_with_no_output_logs_nothing():
flexmock(module.logger).should_receive('log').never()
flexmock(module).should_receive('borg_command').and_return(False)
module.execute_and_log_output(['true'], output_log_level=logging.INFO, shell=False)
module.execute_and_log_output(
['true'], output_log_level=logging.INFO, shell=False, environment=None
)
def test_execute_and_log_output_with_error_exit_status_raises():
flexmock(module.logger).should_receive('log')
flexmock(module).should_receive('borg_command').and_return(False)
with pytest.raises(subprocess.CalledProcessError):
module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
module.execute_and_log_output(
['grep'], output_log_level=logging.INFO, shell=False, environment=None
)

View file

@ -156,11 +156,26 @@ def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
assert exclude_flags == ()
def test_borgmatic_source_directories_set_when_directory_exists():
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module.os.path).should_receive('expanduser')
assert module.borgmatic_source_directories() == [module.BORGMATIC_SOURCE_DIRECTORY]
def test_borgmatic_source_directories_empty_when_directory_does_not_exist():
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module.os.path).should_receive('expanduser')
assert module.borgmatic_source_directories() == []
DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
def test_create_archive_calls_borg_with_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -184,6 +199,7 @@ def test_create_archive_calls_borg_with_parameters():
def test_create_archive_with_patterns_calls_borg_with_patterns():
pattern_flags = ('--patterns-from', 'patterns')
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(
@ -209,6 +225,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
exclude_flags = ('--exclude-from', 'excludes')
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
@ -233,6 +250,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
def test_create_archive_with_log_info_calls_borg_with_info_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -258,6 +276,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -283,6 +302,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -308,6 +328,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -332,6 +353,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -357,6 +379,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_parameter():
# --dry-run and --stats are mutually exclusive, see:
# https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -385,6 +408,7 @@ def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_param
def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_parameter():
# --dry-run and --stats are mutually exclusive, see:
# https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -411,6 +435,7 @@ def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_para
def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -434,6 +459,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -457,6 +483,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
def test_create_archive_with_compression_calls_borg_with_compression_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -480,6 +507,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -503,6 +531,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -526,6 +555,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -549,6 +579,7 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -573,6 +604,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -597,6 +629,7 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -621,6 +654,7 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -645,6 +679,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
def test_create_archive_with_local_path_calls_borg_via_local_path():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -668,6 +703,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -692,6 +728,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
def test_create_archive_with_umask_calls_borg_with_umask_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -714,6 +751,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -736,6 +774,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
def test_create_archive_with_stats_calls_borg_with_stats_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -759,6 +798,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter():
def test_create_archive_with_progress_calls_borg_with_progress_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -782,6 +822,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
def test_create_archive_with_json_calls_borg_with_json_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -807,6 +848,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -833,6 +875,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
def test_create_archive_with_source_directories_glob_expands():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -857,6 +900,7 @@ def test_create_archive_with_source_directories_glob_expands():
def test_create_archive_with_non_matching_source_directories_glob_passes_through():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo*',))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -881,6 +925,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
def test_create_archive_with_glob_calls_borg_with_expanded_directories():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -904,6 +949,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
@ -926,6 +972,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)

View file

@ -23,6 +23,9 @@ def test_run_configuration_runs_actions_for_each_repository():
def test_run_configuration_executes_hooks_for_create_action():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook').twice()
flexmock(module.postgresql).should_receive('dump_databases').once()
flexmock(module.healthchecks).should_receive('ping_healthchecks').twice()
flexmock(module.postgresql).should_receive('remove_database_dumps').once()
flexmock(module).should_receive('run_actions').and_return([])
config = {'location': {'repositories': ['foo']}}
arguments = {'global': flexmock(dry_run=False), 'create': flexmock()}
@ -33,6 +36,8 @@ def test_run_configuration_executes_hooks_for_create_action():
def test_run_configuration_logs_actions_error():
flexmock(module.borg_environment).should_receive('initialize')
flexmock(module.command).should_receive('execute_hook')
flexmock(module.postgresql).should_receive('dump_databases')
flexmock(module.healthchecks).should_receive('ping_healthchecks')
expected_results = [flexmock()]
flexmock(module).should_receive('make_error_log_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').and_raise(OSError)

View file

@ -74,6 +74,27 @@ def test_apply_logical_validation_does_not_raise_otherwise():
module.apply_logical_validation('config.yaml', {'retention': {'keep_secondly': 1000}})
def test_remove_examples_strips_examples_from_map():
schema = {
'map': {
'foo': {'desc': 'thing1', 'example': 'bar'},
'baz': {'desc': 'thing2', 'example': 'quux'},
}
}
module.remove_examples(schema)
assert schema == {'map': {'foo': {'desc': 'thing1'}, 'baz': {'desc': 'thing2'}}}
def test_remove_examples_strips_examples_from_sequence_of_maps():
schema = {'seq': [{'map': {'foo': {'desc': 'thing', 'example': 'bar'}}, 'example': 'stuff'}]}
module.remove_examples(schema)
assert schema == {'seq': [{'map': {'foo': {'desc': 'thing'}}}]}
def test_guard_configuration_contains_repository_does_not_raise_when_repository_in_config():
module.guard_configuration_contains_repository(
repository='repo', configurations={'config.yaml': {'location': {'repositories': ['repo']}}}

View file

@ -8,8 +8,9 @@ from borgmatic import execute as module
def test_execute_command_calls_full_command():
full_command = ['foo', 'bar']
flexmock(module.os, environ={'a': 'b'})
flexmock(module).should_receive('execute_and_log_output').with_args(
full_command, output_log_level=logging.INFO, shell=False
full_command, output_log_level=logging.INFO, shell=False, environment=None
).once()
output = module.execute_command(full_command)
@ -19,8 +20,9 @@ def test_execute_command_calls_full_command():
def test_execute_command_calls_full_command_with_shell():
full_command = ['foo', 'bar']
flexmock(module.os, environ={'a': 'b'})
flexmock(module).should_receive('execute_and_log_output').with_args(
full_command, output_log_level=logging.INFO, shell=True
full_command, output_log_level=logging.INFO, shell=True, environment=None
).once()
output = module.execute_command(full_command, shell=True)
@ -28,11 +30,24 @@ def test_execute_command_calls_full_command_with_shell():
assert output is None
def test_execute_command_calls_full_command_with_extra_environment():
full_command = ['foo', 'bar']
flexmock(module.os, environ={'a': 'b'})
flexmock(module).should_receive('execute_and_log_output').with_args(
full_command, output_log_level=logging.INFO, shell=False, environment={'a': 'b', 'c': 'd'}
).once()
output = module.execute_command(full_command, extra_environment={'c': 'd'})
assert output is None
def test_execute_command_captures_output():
full_command = ['foo', 'bar']
expected_output = '[]'
flexmock(module.os, environ={'a': 'b'})
flexmock(module.subprocess).should_receive('check_output').with_args(
full_command, shell=False
full_command, shell=False, env=None
).and_return(flexmock(decode=lambda: expected_output)).once()
output = module.execute_command(full_command, output_log_level=None)
@ -43,8 +58,9 @@ def test_execute_command_captures_output():
def test_execute_command_captures_output_with_shell():
full_command = ['foo', 'bar']
expected_output = '[]'
flexmock(module.os, environ={'a': 'b'})
flexmock(module.subprocess).should_receive('check_output').with_args(
full_command, shell=True
full_command, shell=True, env=None
).and_return(flexmock(decode=lambda: expected_output)).once()
output = module.execute_command(full_command, output_log_level=None, shell=True)
@ -52,6 +68,21 @@ def test_execute_command_captures_output_with_shell():
assert output == expected_output
def test_execute_command_captures_output_with_extra_environment():
full_command = ['foo', 'bar']
expected_output = '[]'
flexmock(module.os, environ={'a': 'b'})
flexmock(module.subprocess).should_receive('check_output').with_args(
full_command, shell=False, env={'a': 'b', 'c': 'd'}
).and_return(flexmock(decode=lambda: expected_output)).once()
output = module.execute_command(
full_command, output_log_level=None, shell=False, extra_environment={'c': 'd'}
)
assert output == expected_output
def test_execute_command_without_capture_does_not_raise_on_success():
flexmock(module.subprocess).should_receive('check_call').and_raise(
module.subprocess.CalledProcessError(0, 'borg init')