#34: New "extract" consistency check that performs a dry-run extraction of the most recent archive.
This commit is contained in:
parent
525ffa6a28
commit
e85d487c3a
6 changed files with 211 additions and 32 deletions
5
NEWS
5
NEWS
|
@ -1,3 +1,8 @@
|
|||
1.1.5
|
||||
|
||||
* #34: New "extract" consistency check that performs a dry-run extraction of the most recent
|
||||
archive.
|
||||
|
||||
1.1.4
|
||||
|
||||
* #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or
|
||||
|
|
|
@ -77,6 +77,12 @@ default). You should edit the file to suit your needs, as the values are just
|
|||
representative. All fields are optional except where indicated, so feel free
|
||||
to remove anything you don't need.
|
||||
|
||||
You can also have a look at the [full configuration
|
||||
schema](https://torsion.org/hg/borgmatic/file/tip/borgmatic/config/schema.yaml)
|
||||
for the authoritative set of all configuration options. This is handy if
|
||||
borgmatic has added new options since you originally created your
|
||||
configuration file.
|
||||
|
||||
|
||||
### Multiple configuration files
|
||||
|
||||
|
|
|
@ -3,6 +3,7 @@ import glob
|
|||
import itertools
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
@ -43,7 +44,7 @@ def create_archive(
|
|||
):
|
||||
'''
|
||||
Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
|
||||
repository path, a list of exclude patterns, and a command to run, create an attic archive.
|
||||
repository path, a list of exclude patterns, and a command to run, create a Borg archive.
|
||||
'''
|
||||
sources = tuple(
|
||||
itertools.chain.from_iterable(
|
||||
|
@ -68,8 +69,8 @@ def create_archive(
|
|||
|
||||
full_command = (
|
||||
command, 'create',
|
||||
'{repo}::{hostname}-{timestamp}'.format(
|
||||
repo=repository,
|
||||
'{repository}::{hostname}-{timestamp}'.format(
|
||||
repository=repository,
|
||||
hostname=platform.node(),
|
||||
timestamp=datetime.now().isoformat(),
|
||||
),
|
||||
|
@ -104,7 +105,7 @@ def _make_prune_flags(retention_config):
|
|||
def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None):
|
||||
'''
|
||||
Given a verbosity flag, a local or remote repository path, a retention config dict, and a
|
||||
command to run, prune attic archives according the the retention policy specified in that
|
||||
command to run, prune Borg archives according the the retention policy specified in that
|
||||
configuration.
|
||||
'''
|
||||
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
||||
|
@ -170,21 +171,21 @@ def _make_check_flags(checks, check_last=None):
|
|||
|
||||
return tuple(
|
||||
'--{}-only'.format(check) for check in checks
|
||||
if check in DEFAULT_CHECKS
|
||||
) + last_flag
|
||||
|
||||
|
||||
def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None):
|
||||
'''
|
||||
Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
|
||||
command to run, check the contained attic archives for consistency.
|
||||
command to run, check the contained Borg archives for consistency.
|
||||
|
||||
If there are no consistency checks to run, skip running them.
|
||||
'''
|
||||
checks = _parse_checks(consistency_config)
|
||||
check_last = consistency_config.get('check_last', None)
|
||||
if not checks:
|
||||
return
|
||||
|
||||
if set(checks).intersection(set(DEFAULT_CHECKS)):
|
||||
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
||||
verbosity_flags = {
|
||||
VERBOSITY_SOME: ('--info',),
|
||||
|
@ -200,3 +201,42 @@ def check_archives(verbosity, repository, consistency_config, command=COMMAND, r
|
|||
stdout = None if verbosity_flags else open(os.devnull, 'w')
|
||||
|
||||
subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
|
||||
|
||||
if 'extract' in checks:
|
||||
extract_last_archive_dry_run(verbosity, repository, command, remote_path)
|
||||
|
||||
|
||||
def extract_last_archive_dry_run(verbosity, repository, command=COMMAND, remote_path=None):
|
||||
'''
|
||||
Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
|
||||
the dry-run.
|
||||
'''
|
||||
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
||||
verbosity_flags = {
|
||||
VERBOSITY_SOME: ('--info',),
|
||||
VERBOSITY_LOTS: ('--debug',),
|
||||
}.get(verbosity, ())
|
||||
|
||||
full_list_command = (
|
||||
command, 'list',
|
||||
'--short',
|
||||
repository,
|
||||
) + remote_path_flags + verbosity_flags
|
||||
|
||||
list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
|
||||
|
||||
last_archive_name = list_output.strip().split('\n')[-1]
|
||||
if not last_archive_name:
|
||||
return
|
||||
|
||||
list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else ()
|
||||
full_extract_command = (
|
||||
command, 'extract',
|
||||
'--dry-run',
|
||||
'{repository}::{last_archive_name}'.format(
|
||||
repository=repository,
|
||||
last_archive_name=last_archive_name,
|
||||
),
|
||||
) + remote_path_flags + verbosity_flags + list_flag
|
||||
|
||||
subprocess.check_call(full_extract_command)
|
||||
|
|
|
@ -106,21 +106,25 @@ map:
|
|||
consistency:
|
||||
desc: |
|
||||
Consistency checks to run after backups. See
|
||||
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
|
||||
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and
|
||||
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details.
|
||||
map:
|
||||
checks:
|
||||
seq:
|
||||
- type: str
|
||||
enum: ['repository', 'archives', 'disabled']
|
||||
enum: ['repository', 'archives', 'extract', 'disabled']
|
||||
unique: true
|
||||
desc: |
|
||||
List of consistency checks to run: "repository", "archives", or both. Defaults
|
||||
to both. Set to "disabled" to disable all consistency checks. See
|
||||
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
|
||||
List of one or more consistency checks to run: "repository", "archives", and/or
|
||||
"extract". Defaults to "repository" and "archives". Set to "disabled" to disable
|
||||
all consistency checks. "repository" checks the consistency of the repository,
|
||||
"archive" checks all of the archives, and "extract" does an extraction dry-run
|
||||
of just the most recent archive.
|
||||
example:
|
||||
- repository
|
||||
- archives
|
||||
check_last:
|
||||
type: int
|
||||
desc: Restrict the number of checked archives to the last n.
|
||||
desc: Restrict the number of checked archives to the last n. Applies only to the
|
||||
"archives" check.
|
||||
example: 3
|
||||
|
|
|
@ -4,6 +4,7 @@ import sys
|
|||
import os
|
||||
|
||||
from flexmock import flexmock
|
||||
import pytest
|
||||
|
||||
from borgmatic import borg as module
|
||||
from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
|
||||
|
@ -46,17 +47,23 @@ def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
|
|||
|
||||
|
||||
def insert_subprocess_mock(check_call_command, **kwargs):
|
||||
subprocess = flexmock(STDOUT=STDOUT)
|
||||
subprocess = flexmock(module.subprocess)
|
||||
subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
|
||||
flexmock(module).subprocess = subprocess
|
||||
|
||||
|
||||
def insert_subprocess_never():
|
||||
subprocess = flexmock()
|
||||
subprocess = flexmock(module.subprocess)
|
||||
subprocess.should_receive('check_call').never()
|
||||
flexmock(module).subprocess = subprocess
|
||||
|
||||
|
||||
def insert_subprocess_check_output_mock(check_output_command, result, **kwargs):
|
||||
subprocess = flexmock(module.subprocess)
|
||||
subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once()
|
||||
flexmock(module).subprocess = subprocess
|
||||
|
||||
|
||||
def insert_platform_mock():
|
||||
flexmock(module.platform).should_receive('node').and_return('host')
|
||||
|
||||
|
@ -395,9 +402,15 @@ def test_parse_checks_with_disabled_returns_no_checks():
|
|||
|
||||
|
||||
def test_make_check_flags_with_checks_returns_flags():
|
||||
flags = module._make_check_flags(('foo', 'bar'))
|
||||
flags = module._make_check_flags(('repository',))
|
||||
|
||||
assert flags == ('--foo-only', '--bar-only')
|
||||
assert flags == ('--repository-only',)
|
||||
|
||||
|
||||
def test_make_check_flags_with_extract_check_does_not_make_extract_flag():
|
||||
flags = module._make_check_flags(('extract',))
|
||||
|
||||
assert flags == ()
|
||||
|
||||
|
||||
def test_make_check_flags_with_default_checks_returns_no_flags():
|
||||
|
@ -407,19 +420,27 @@ def test_make_check_flags_with_default_checks_returns_no_flags():
|
|||
|
||||
|
||||
def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
|
||||
flags = module._make_check_flags(('foo', 'bar'), check_last=3)
|
||||
flags = module._make_check_flags(('repository',), check_last=3)
|
||||
|
||||
assert flags == ('--foo-only', '--bar-only', '--last', '3')
|
||||
assert flags == ('--repository-only', '--last', '3')
|
||||
|
||||
|
||||
def test_make_check_flags_with_last_returns_last_flag():
|
||||
def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
|
||||
flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
|
||||
|
||||
assert flags == ('--last', '3')
|
||||
|
||||
|
||||
def test_check_archives_should_call_borg_with_parameters():
|
||||
checks = flexmock()
|
||||
@pytest.mark.parametrize(
|
||||
'checks',
|
||||
(
|
||||
('repository',),
|
||||
('archives',),
|
||||
('repository', 'archives'),
|
||||
('repository', 'archives', 'other'),
|
||||
),
|
||||
)
|
||||
def test_check_archives_should_call_borg_with_parameters(checks):
|
||||
check_last = flexmock()
|
||||
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||
|
@ -442,9 +463,27 @@ def test_check_archives_should_call_borg_with_parameters():
|
|||
)
|
||||
|
||||
|
||||
def test_check_archives_with_extract_check_should_call_extract_only():
|
||||
checks = ('extract',)
|
||||
check_last = flexmock()
|
||||
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||
flexmock(module).should_receive('_make_check_flags').never()
|
||||
flexmock(module).should_receive('extract_last_archive_dry_run').once()
|
||||
insert_subprocess_never()
|
||||
|
||||
module.check_archives(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
consistency_config=consistency_config,
|
||||
command='borg',
|
||||
)
|
||||
|
||||
|
||||
def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter():
|
||||
checks = ('repository',)
|
||||
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
||||
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
|
||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||
flexmock(module).should_receive('_make_check_flags').and_return(())
|
||||
insert_subprocess_mock(
|
||||
('borg', 'check', 'repo', '--info'),
|
||||
|
@ -462,8 +501,9 @@ def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter
|
|||
|
||||
|
||||
def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
|
||||
checks = ('repository',)
|
||||
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
||||
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
|
||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||
flexmock(module).should_receive('_make_check_flags').and_return(())
|
||||
insert_subprocess_mock(
|
||||
('borg', 'check', 'repo', '--debug'),
|
||||
|
@ -494,7 +534,7 @@ def test_check_archives_without_any_checks_should_bail():
|
|||
|
||||
|
||||
def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
|
||||
checks = flexmock()
|
||||
checks = ('repository',)
|
||||
check_last = flexmock()
|
||||
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||
|
@ -516,3 +556,87 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param
|
|||
command='borg',
|
||||
remote_path='borg1',
|
||||
)
|
||||
|
||||
|
||||
def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
|
||||
flexmock(sys.stdout).encoding = 'utf-8'
|
||||
insert_subprocess_check_output_mock(
|
||||
('borg', 'list', '--short', 'repo'),
|
||||
result='archive1\narchive2\n'.encode('utf-8'),
|
||||
)
|
||||
insert_subprocess_mock(
|
||||
('borg', 'extract', '--dry-run', 'repo::archive2'),
|
||||
)
|
||||
|
||||
module.extract_last_archive_dry_run(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
command='borg',
|
||||
)
|
||||
|
||||
|
||||
def test_extract_last_archive_dry_run_without_any_archives_should_bail():
|
||||
flexmock(sys.stdout).encoding = 'utf-8'
|
||||
insert_subprocess_check_output_mock(
|
||||
('borg', 'list', '--short', 'repo'),
|
||||
result='\n'.encode('utf-8'),
|
||||
)
|
||||
insert_subprocess_never()
|
||||
|
||||
module.extract_last_archive_dry_run(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
command='borg',
|
||||
)
|
||||
|
||||
|
||||
def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter():
|
||||
flexmock(sys.stdout).encoding = 'utf-8'
|
||||
insert_subprocess_check_output_mock(
|
||||
('borg', 'list', '--short', 'repo', '--info'),
|
||||
result='archive1\narchive2\n'.encode('utf-8'),
|
||||
)
|
||||
insert_subprocess_mock(
|
||||
('borg', 'extract', '--dry-run', 'repo::archive2', '--info'),
|
||||
)
|
||||
|
||||
module.extract_last_archive_dry_run(
|
||||
verbosity=VERBOSITY_SOME,
|
||||
repository='repo',
|
||||
command='borg',
|
||||
)
|
||||
|
||||
|
||||
def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter():
|
||||
flexmock(sys.stdout).encoding = 'utf-8'
|
||||
insert_subprocess_check_output_mock(
|
||||
('borg', 'list', '--short', 'repo', '--debug'),
|
||||
result='archive1\narchive2\n'.encode('utf-8'),
|
||||
)
|
||||
insert_subprocess_mock(
|
||||
('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'),
|
||||
)
|
||||
|
||||
module.extract_last_archive_dry_run(
|
||||
verbosity=VERBOSITY_LOTS,
|
||||
repository='repo',
|
||||
command='borg',
|
||||
)
|
||||
|
||||
|
||||
def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters():
|
||||
flexmock(sys.stdout).encoding = 'utf-8'
|
||||
insert_subprocess_check_output_mock(
|
||||
('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'),
|
||||
result='archive1\narchive2\n'.encode('utf-8'),
|
||||
)
|
||||
insert_subprocess_mock(
|
||||
('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'),
|
||||
)
|
||||
|
||||
module.extract_last_archive_dry_run(
|
||||
verbosity=None,
|
||||
repository='repo',
|
||||
command='borg',
|
||||
remote_path='borg1',
|
||||
)
|
||||
|
|
2
setup.py
2
setup.py
|
@ -1,7 +1,7 @@
|
|||
from setuptools import setup, find_packages
|
||||
|
||||
|
||||
VERSION = '1.1.4'
|
||||
VERSION = '1.1.5'
|
||||
|
||||
|
||||
setup(
|
||||
|
|
Loading…
Reference in a new issue