From e85d487c3a3f7fbc13609cd334a6f0a1614096c0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 30 Jul 2017 11:16:26 -0700 Subject: [PATCH] #34: New "extract" consistency check that performs a dry-run extraction of the most recent archive. --- NEWS | 5 + README.md | 6 ++ borgmatic/borg.py | 66 ++++++++++--- borgmatic/config/schema.yaml | 16 ++-- borgmatic/tests/unit/test_borg.py | 148 +++++++++++++++++++++++++++--- setup.py | 2 +- 6 files changed, 211 insertions(+), 32 deletions(-) diff --git a/NEWS b/NEWS index c5a8162..6222fea 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.1.5 + + * #34: New "extract" consistency check that performs a dry-run extraction of the most recent + archive. + 1.1.4 * #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or diff --git a/README.md b/README.md index 22b1da7..057388b 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,12 @@ default). You should edit the file to suit your needs, as the values are just representative. All fields are optional except where indicated, so feel free to remove anything you don't need. +You can also have a look at the [full configuration +schema](https://torsion.org/hg/borgmatic/file/tip/borgmatic/config/schema.yaml) +for the authoritative set of all configuration options. This is handy if +borgmatic has added new options since you originally created your +configuration file. + ### Multiple configuration files diff --git a/borgmatic/borg.py b/borgmatic/borg.py index 8c4c730..fff6e7e 100644 --- a/borgmatic/borg.py +++ b/borgmatic/borg.py @@ -3,6 +3,7 @@ import glob import itertools import os import platform +import sys import re import subprocess import tempfile @@ -43,7 +44,7 @@ def create_archive( ): ''' Given a vebosity flag, a storage config dict, a list of source directories, a local or remote - repository path, a list of exclude patterns, and a command to run, create an attic archive. + repository path, a list of exclude patterns, and a command to run, create a Borg archive. ''' sources = tuple( itertools.chain.from_iterable( @@ -68,8 +69,8 @@ def create_archive( full_command = ( command, 'create', - '{repo}::{hostname}-{timestamp}'.format( - repo=repository, + '{repository}::{hostname}-{timestamp}'.format( + repository=repository, hostname=platform.node(), timestamp=datetime.now().isoformat(), ), @@ -104,7 +105,7 @@ def _make_prune_flags(retention_config): def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None): ''' Given a verbosity flag, a local or remote repository path, a retention config dict, and a - command to run, prune attic archives according the the retention policy specified in that + command to run, prune Borg archives according the the retention policy specified in that configuration. ''' remote_path_flags = ('--remote-path', remote_path) if remote_path else () @@ -170,33 +171,72 @@ def _make_check_flags(checks, check_last=None): return tuple( '--{}-only'.format(check) for check in checks + if check in DEFAULT_CHECKS ) + last_flag def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None): ''' Given a verbosity flag, a local or remote repository path, a consistency config dict, and a - command to run, check the contained attic archives for consistency. + command to run, check the contained Borg archives for consistency. If there are no consistency checks to run, skip running them. ''' checks = _parse_checks(consistency_config) check_last = consistency_config.get('check_last', None) - if not checks: - return + if set(checks).intersection(set(DEFAULT_CHECKS)): + remote_path_flags = ('--remote-path', remote_path) if remote_path else () + verbosity_flags = { + VERBOSITY_SOME: ('--info',), + VERBOSITY_LOTS: ('--debug',), + }.get(verbosity, ()) + + full_command = ( + command, 'check', + repository, + ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags + + # The check command spews to stdout/stderr even without the verbose flag. Suppress it. + stdout = None if verbosity_flags else open(os.devnull, 'w') + + subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) + + if 'extract' in checks: + extract_last_archive_dry_run(verbosity, repository, command, remote_path) + + +def extract_last_archive_dry_run(verbosity, repository, command=COMMAND, remote_path=None): + ''' + Perform an extraction dry-run of just the most recent archive. If there are no archives, skip + the dry-run. + ''' remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { VERBOSITY_SOME: ('--info',), VERBOSITY_LOTS: ('--debug',), }.get(verbosity, ()) - full_command = ( - command, 'check', + full_list_command = ( + command, 'list', + '--short', repository, - ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags + ) + remote_path_flags + verbosity_flags - # The check command spews to stdout/stderr even without the verbose flag. Suppress it. - stdout = None if verbosity_flags else open(os.devnull, 'w') + list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding) - subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) + last_archive_name = list_output.strip().split('\n')[-1] + if not last_archive_name: + return + + list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else () + full_extract_command = ( + command, 'extract', + '--dry-run', + '{repository}::{last_archive_name}'.format( + repository=repository, + last_archive_name=last_archive_name, + ), + ) + remote_path_flags + verbosity_flags + list_flag + + subprocess.check_call(full_extract_command) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 07e245d..567ec73 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -106,21 +106,25 @@ map: consistency: desc: | Consistency checks to run after backups. See - https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details. map: checks: seq: - type: str - enum: ['repository', 'archives', 'disabled'] + enum: ['repository', 'archives', 'extract', 'disabled'] unique: true desc: | - List of consistency checks to run: "repository", "archives", or both. Defaults - to both. Set to "disabled" to disable all consistency checks. See - https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. + List of one or more consistency checks to run: "repository", "archives", and/or + "extract". Defaults to "repository" and "archives". Set to "disabled" to disable + all consistency checks. "repository" checks the consistency of the repository, + "archive" checks all of the archives, and "extract" does an extraction dry-run + of just the most recent archive. example: - repository - archives check_last: type: int - desc: Restrict the number of checked archives to the last n. + desc: Restrict the number of checked archives to the last n. Applies only to the + "archives" check. example: 3 diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 4963ab5..1cba040 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -4,6 +4,7 @@ import sys import os from flexmock import flexmock +import pytest from borgmatic import borg as module from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS @@ -46,17 +47,23 @@ def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise(): def insert_subprocess_mock(check_call_command, **kwargs): - subprocess = flexmock(STDOUT=STDOUT) + subprocess = flexmock(module.subprocess) subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() flexmock(module).subprocess = subprocess def insert_subprocess_never(): - subprocess = flexmock() + subprocess = flexmock(module.subprocess) subprocess.should_receive('check_call').never() flexmock(module).subprocess = subprocess +def insert_subprocess_check_output_mock(check_output_command, result, **kwargs): + subprocess = flexmock(module.subprocess) + subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once() + flexmock(module).subprocess = subprocess + + def insert_platform_mock(): flexmock(module.platform).should_receive('node').and_return('host') @@ -395,9 +402,15 @@ def test_parse_checks_with_disabled_returns_no_checks(): def test_make_check_flags_with_checks_returns_flags(): - flags = module._make_check_flags(('foo', 'bar')) + flags = module._make_check_flags(('repository',)) - assert flags == ('--foo-only', '--bar-only') + assert flags == ('--repository-only',) + + +def test_make_check_flags_with_extract_check_does_not_make_extract_flag(): + flags = module._make_check_flags(('extract',)) + + assert flags == () def test_make_check_flags_with_default_checks_returns_no_flags(): @@ -407,19 +420,27 @@ def test_make_check_flags_with_default_checks_returns_no_flags(): def test_make_check_flags_with_checks_and_last_returns_flags_including_last(): - flags = module._make_check_flags(('foo', 'bar'), check_last=3) + flags = module._make_check_flags(('repository',), check_last=3) - assert flags == ('--foo-only', '--bar-only', '--last', '3') + assert flags == ('--repository-only', '--last', '3') -def test_make_check_flags_with_last_returns_last_flag(): +def test_make_check_flags_with_default_checks_and_last_returns_last_flag(): flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3) assert flags == ('--last', '3') -def test_check_archives_should_call_borg_with_parameters(): - checks = flexmock() +@pytest.mark.parametrize( + 'checks', + ( + ('repository',), + ('archives',), + ('repository', 'archives'), + ('repository', 'archives', 'other'), + ), +) +def test_check_archives_should_call_borg_with_parameters(checks): check_last = flexmock() consistency_config = flexmock().should_receive('get').and_return(check_last).mock flexmock(module).should_receive('_parse_checks').and_return(checks) @@ -442,9 +463,27 @@ def test_check_archives_should_call_borg_with_parameters(): ) +def test_check_archives_with_extract_check_should_call_extract_only(): + checks = ('extract',) + check_last = flexmock() + consistency_config = flexmock().should_receive('get').and_return(check_last).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').never() + flexmock(module).should_receive('extract_last_archive_dry_run').once() + insert_subprocess_never() + + module.check_archives( + verbosity=None, + repository='repo', + consistency_config=consistency_config, + command='borg', + ) + + def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter(): + checks = ('repository',) consistency_config = flexmock().should_receive('get').and_return(None).mock - flexmock(module).should_receive('_parse_checks').and_return(flexmock()) + flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( ('borg', 'check', 'repo', '--info'), @@ -462,8 +501,9 @@ def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter(): + checks = ('repository',) consistency_config = flexmock().should_receive('get').and_return(None).mock - flexmock(module).should_receive('_parse_checks').and_return(flexmock()) + flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( ('borg', 'check', 'repo', '--debug'), @@ -494,7 +534,7 @@ def test_check_archives_without_any_checks_should_bail(): def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters(): - checks = flexmock() + checks = ('repository',) check_last = flexmock() consistency_config = flexmock().should_receive('get').and_return(check_last).mock flexmock(module).should_receive('_parse_checks').and_return(checks) @@ -516,3 +556,87 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param command='borg', remote_path='borg1', ) + + +def test_extract_last_archive_dry_run_should_call_borg_with_last_archive(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2'), + ) + + module.extract_last_archive_dry_run( + verbosity=None, + repository='repo', + command='borg', + ) + + +def test_extract_last_archive_dry_run_without_any_archives_should_bail(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo'), + result='\n'.encode('utf-8'), + ) + insert_subprocess_never() + + module.extract_last_archive_dry_run( + verbosity=None, + repository='repo', + command='borg', + ) + + +def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo', '--info'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2', '--info'), + ) + + module.extract_last_archive_dry_run( + verbosity=VERBOSITY_SOME, + repository='repo', + command='borg', + ) + + +def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo', '--debug'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'), + ) + + module.extract_last_archive_dry_run( + verbosity=VERBOSITY_LOTS, + repository='repo', + command='borg', + ) + + +def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'), + ) + + module.extract_last_archive_dry_run( + verbosity=None, + repository='repo', + command='borg', + remote_path='borg1', + ) diff --git a/setup.py b/setup.py index d4f587b..2ff9690 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.4' +VERSION = '1.1.5' setup(