#34: New "extract" consistency check that performs a dry-run extraction of the most recent archive.

This commit is contained in:
Dan Helfman 2017-07-30 11:16:26 -07:00
parent 23679a6edd
commit 9bea7ae5ed
6 changed files with 211 additions and 32 deletions

5
NEWS
View file

@ -1,3 +1,8 @@
1.1.5
* #34: New "extract" consistency check that performs a dry-run extraction of the most recent
archive.
1.1.4 1.1.4
* #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or * #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or

View file

@ -77,6 +77,12 @@ default). You should edit the file to suit your needs, as the values are just
representative. All fields are optional except where indicated, so feel free representative. All fields are optional except where indicated, so feel free
to remove anything you don't need. to remove anything you don't need.
You can also have a look at the [full configuration
schema](https://torsion.org/hg/borgmatic/file/tip/borgmatic/config/schema.yaml)
for the authoritative set of all configuration options. This is handy if
borgmatic has added new options since you originally created your
configuration file.
### Multiple configuration files ### Multiple configuration files

View file

@ -3,6 +3,7 @@ import glob
import itertools import itertools
import os import os
import platform import platform
import sys
import re import re
import subprocess import subprocess
import tempfile import tempfile
@ -43,7 +44,7 @@ def create_archive(
): ):
''' '''
Given a vebosity flag, a storage config dict, a list of source directories, a local or remote Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
repository path, a list of exclude patterns, and a command to run, create an attic archive. repository path, a list of exclude patterns, and a command to run, create a Borg archive.
''' '''
sources = tuple( sources = tuple(
itertools.chain.from_iterable( itertools.chain.from_iterable(
@ -68,8 +69,8 @@ def create_archive(
full_command = ( full_command = (
command, 'create', command, 'create',
'{repo}::{hostname}-{timestamp}'.format( '{repository}::{hostname}-{timestamp}'.format(
repo=repository, repository=repository,
hostname=platform.node(), hostname=platform.node(),
timestamp=datetime.now().isoformat(), timestamp=datetime.now().isoformat(),
), ),
@ -104,7 +105,7 @@ def _make_prune_flags(retention_config):
def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None): def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None):
''' '''
Given a verbosity flag, a local or remote repository path, a retention config dict, and a Given a verbosity flag, a local or remote repository path, a retention config dict, and a
command to run, prune attic archives according the the retention policy specified in that command to run, prune Borg archives according the the retention policy specified in that
configuration. configuration.
''' '''
remote_path_flags = ('--remote-path', remote_path) if remote_path else () remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
@ -170,21 +171,21 @@ def _make_check_flags(checks, check_last=None):
return tuple( return tuple(
'--{}-only'.format(check) for check in checks '--{}-only'.format(check) for check in checks
if check in DEFAULT_CHECKS
) + last_flag ) + last_flag
def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None): def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None):
''' '''
Given a verbosity flag, a local or remote repository path, a consistency config dict, and a Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
command to run, check the contained attic archives for consistency. command to run, check the contained Borg archives for consistency.
If there are no consistency checks to run, skip running them. If there are no consistency checks to run, skip running them.
''' '''
checks = _parse_checks(consistency_config) checks = _parse_checks(consistency_config)
check_last = consistency_config.get('check_last', None) check_last = consistency_config.get('check_last', None)
if not checks:
return
if set(checks).intersection(set(DEFAULT_CHECKS)):
remote_path_flags = ('--remote-path', remote_path) if remote_path else () remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
verbosity_flags = { verbosity_flags = {
VERBOSITY_SOME: ('--info',), VERBOSITY_SOME: ('--info',),
@ -200,3 +201,42 @@ def check_archives(verbosity, repository, consistency_config, command=COMMAND, r
stdout = None if verbosity_flags else open(os.devnull, 'w') stdout = None if verbosity_flags else open(os.devnull, 'w')
subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
if 'extract' in checks:
extract_last_archive_dry_run(verbosity, repository, command, remote_path)
def extract_last_archive_dry_run(verbosity, repository, command=COMMAND, remote_path=None):
'''
Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
the dry-run.
'''
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
verbosity_flags = {
VERBOSITY_SOME: ('--info',),
VERBOSITY_LOTS: ('--debug',),
}.get(verbosity, ())
full_list_command = (
command, 'list',
'--short',
repository,
) + remote_path_flags + verbosity_flags
list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
last_archive_name = list_output.strip().split('\n')[-1]
if not last_archive_name:
return
list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else ()
full_extract_command = (
command, 'extract',
'--dry-run',
'{repository}::{last_archive_name}'.format(
repository=repository,
last_archive_name=last_archive_name,
),
) + remote_path_flags + verbosity_flags + list_flag
subprocess.check_call(full_extract_command)

View file

@ -106,21 +106,25 @@ map:
consistency: consistency:
desc: | desc: |
Consistency checks to run after backups. See Consistency checks to run after backups. See
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details.
map: map:
checks: checks:
seq: seq:
- type: str - type: str
enum: ['repository', 'archives', 'disabled'] enum: ['repository', 'archives', 'extract', 'disabled']
unique: true unique: true
desc: | desc: |
List of consistency checks to run: "repository", "archives", or both. Defaults List of one or more consistency checks to run: "repository", "archives", and/or
to both. Set to "disabled" to disable all consistency checks. See "extract". Defaults to "repository" and "archives". Set to "disabled" to disable
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. all consistency checks. "repository" checks the consistency of the repository,
"archive" checks all of the archives, and "extract" does an extraction dry-run
of just the most recent archive.
example: example:
- repository - repository
- archives - archives
check_last: check_last:
type: int type: int
desc: Restrict the number of checked archives to the last n. desc: Restrict the number of checked archives to the last n. Applies only to the
"archives" check.
example: 3 example: 3

View file

@ -4,6 +4,7 @@ import sys
import os import os
from flexmock import flexmock from flexmock import flexmock
import pytest
from borgmatic import borg as module from borgmatic import borg as module
from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
@ -46,17 +47,23 @@ def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
def insert_subprocess_mock(check_call_command, **kwargs): def insert_subprocess_mock(check_call_command, **kwargs):
subprocess = flexmock(STDOUT=STDOUT) subprocess = flexmock(module.subprocess)
subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
flexmock(module).subprocess = subprocess flexmock(module).subprocess = subprocess
def insert_subprocess_never(): def insert_subprocess_never():
subprocess = flexmock() subprocess = flexmock(module.subprocess)
subprocess.should_receive('check_call').never() subprocess.should_receive('check_call').never()
flexmock(module).subprocess = subprocess flexmock(module).subprocess = subprocess
def insert_subprocess_check_output_mock(check_output_command, result, **kwargs):
subprocess = flexmock(module.subprocess)
subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once()
flexmock(module).subprocess = subprocess
def insert_platform_mock(): def insert_platform_mock():
flexmock(module.platform).should_receive('node').and_return('host') flexmock(module.platform).should_receive('node').and_return('host')
@ -395,9 +402,15 @@ def test_parse_checks_with_disabled_returns_no_checks():
def test_make_check_flags_with_checks_returns_flags(): def test_make_check_flags_with_checks_returns_flags():
flags = module._make_check_flags(('foo', 'bar')) flags = module._make_check_flags(('repository',))
assert flags == ('--foo-only', '--bar-only') assert flags == ('--repository-only',)
def test_make_check_flags_with_extract_check_does_not_make_extract_flag():
flags = module._make_check_flags(('extract',))
assert flags == ()
def test_make_check_flags_with_default_checks_returns_no_flags(): def test_make_check_flags_with_default_checks_returns_no_flags():
@ -407,19 +420,27 @@ def test_make_check_flags_with_default_checks_returns_no_flags():
def test_make_check_flags_with_checks_and_last_returns_flags_including_last(): def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
flags = module._make_check_flags(('foo', 'bar'), check_last=3) flags = module._make_check_flags(('repository',), check_last=3)
assert flags == ('--foo-only', '--bar-only', '--last', '3') assert flags == ('--repository-only', '--last', '3')
def test_make_check_flags_with_last_returns_last_flag(): def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3) flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
assert flags == ('--last', '3') assert flags == ('--last', '3')
def test_check_archives_should_call_borg_with_parameters(): @pytest.mark.parametrize(
checks = flexmock() 'checks',
(
('repository',),
('archives',),
('repository', 'archives'),
('repository', 'archives', 'other'),
),
)
def test_check_archives_should_call_borg_with_parameters(checks):
check_last = flexmock() check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock consistency_config = flexmock().should_receive('get').and_return(check_last).mock
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
@ -442,9 +463,27 @@ def test_check_archives_should_call_borg_with_parameters():
) )
def test_check_archives_with_extract_check_should_call_extract_only():
checks = ('extract',)
check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').never()
flexmock(module).should_receive('extract_last_archive_dry_run').once()
insert_subprocess_never()
module.check_archives(
verbosity=None,
repository='repo',
consistency_config=consistency_config,
command='borg',
)
def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter(): def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter():
checks = ('repository',)
consistency_config = flexmock().should_receive('get').and_return(None).mock consistency_config = flexmock().should_receive('get').and_return(None).mock
flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').and_return(()) flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock( insert_subprocess_mock(
('borg', 'check', 'repo', '--info'), ('borg', 'check', 'repo', '--info'),
@ -462,8 +501,9 @@ def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter
def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter(): def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
checks = ('repository',)
consistency_config = flexmock().should_receive('get').and_return(None).mock consistency_config = flexmock().should_receive('get').and_return(None).mock
flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_parse_checks').and_return(checks)
flexmock(module).should_receive('_make_check_flags').and_return(()) flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock( insert_subprocess_mock(
('borg', 'check', 'repo', '--debug'), ('borg', 'check', 'repo', '--debug'),
@ -494,7 +534,7 @@ def test_check_archives_without_any_checks_should_bail():
def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters(): def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
checks = flexmock() checks = ('repository',)
check_last = flexmock() check_last = flexmock()
consistency_config = flexmock().should_receive('get').and_return(check_last).mock consistency_config = flexmock().should_receive('get').and_return(check_last).mock
flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_parse_checks').and_return(checks)
@ -516,3 +556,87 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param
command='borg', command='borg',
remote_path='borg1', remote_path='borg1',
) )
def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
flexmock(sys.stdout).encoding = 'utf-8'
insert_subprocess_check_output_mock(
('borg', 'list', '--short', 'repo'),
result='archive1\narchive2\n'.encode('utf-8'),
)
insert_subprocess_mock(
('borg', 'extract', '--dry-run', 'repo::archive2'),
)
module.extract_last_archive_dry_run(
verbosity=None,
repository='repo',
command='borg',
)
def test_extract_last_archive_dry_run_without_any_archives_should_bail():
flexmock(sys.stdout).encoding = 'utf-8'
insert_subprocess_check_output_mock(
('borg', 'list', '--short', 'repo'),
result='\n'.encode('utf-8'),
)
insert_subprocess_never()
module.extract_last_archive_dry_run(
verbosity=None,
repository='repo',
command='borg',
)
def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter():
flexmock(sys.stdout).encoding = 'utf-8'
insert_subprocess_check_output_mock(
('borg', 'list', '--short', 'repo', '--info'),
result='archive1\narchive2\n'.encode('utf-8'),
)
insert_subprocess_mock(
('borg', 'extract', '--dry-run', 'repo::archive2', '--info'),
)
module.extract_last_archive_dry_run(
verbosity=VERBOSITY_SOME,
repository='repo',
command='borg',
)
def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter():
flexmock(sys.stdout).encoding = 'utf-8'
insert_subprocess_check_output_mock(
('borg', 'list', '--short', 'repo', '--debug'),
result='archive1\narchive2\n'.encode('utf-8'),
)
insert_subprocess_mock(
('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'),
)
module.extract_last_archive_dry_run(
verbosity=VERBOSITY_LOTS,
repository='repo',
command='borg',
)
def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters():
flexmock(sys.stdout).encoding = 'utf-8'
insert_subprocess_check_output_mock(
('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'),
result='archive1\narchive2\n'.encode('utf-8'),
)
insert_subprocess_mock(
('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'),
)
module.extract_last_archive_dry_run(
verbosity=None,
repository='repo',
command='borg',
remote_path='borg1',
)

View file

@ -1,7 +1,7 @@
from setuptools import setup, find_packages from setuptools import setup, find_packages
VERSION = '1.1.4' VERSION = '1.1.5'
setup( setup(