#34: New "extract" consistency check that performs a dry-run extraction of the most recent archive.
This commit is contained in:
parent
525ffa6a28
commit
e85d487c3a
6 changed files with 211 additions and 32 deletions
5
NEWS
5
NEWS
|
@ -1,3 +1,8 @@
|
||||||
|
1.1.5
|
||||||
|
|
||||||
|
* #34: New "extract" consistency check that performs a dry-run extraction of the most recent
|
||||||
|
archive.
|
||||||
|
|
||||||
1.1.4
|
1.1.4
|
||||||
|
|
||||||
* #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or
|
* #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or
|
||||||
|
|
|
@ -77,6 +77,12 @@ default). You should edit the file to suit your needs, as the values are just
|
||||||
representative. All fields are optional except where indicated, so feel free
|
representative. All fields are optional except where indicated, so feel free
|
||||||
to remove anything you don't need.
|
to remove anything you don't need.
|
||||||
|
|
||||||
|
You can also have a look at the [full configuration
|
||||||
|
schema](https://torsion.org/hg/borgmatic/file/tip/borgmatic/config/schema.yaml)
|
||||||
|
for the authoritative set of all configuration options. This is handy if
|
||||||
|
borgmatic has added new options since you originally created your
|
||||||
|
configuration file.
|
||||||
|
|
||||||
|
|
||||||
### Multiple configuration files
|
### Multiple configuration files
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,7 @@ import glob
|
||||||
import itertools
|
import itertools
|
||||||
import os
|
import os
|
||||||
import platform
|
import platform
|
||||||
|
import sys
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
|
@ -43,7 +44,7 @@ def create_archive(
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
|
Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
|
||||||
repository path, a list of exclude patterns, and a command to run, create an attic archive.
|
repository path, a list of exclude patterns, and a command to run, create a Borg archive.
|
||||||
'''
|
'''
|
||||||
sources = tuple(
|
sources = tuple(
|
||||||
itertools.chain.from_iterable(
|
itertools.chain.from_iterable(
|
||||||
|
@ -68,8 +69,8 @@ def create_archive(
|
||||||
|
|
||||||
full_command = (
|
full_command = (
|
||||||
command, 'create',
|
command, 'create',
|
||||||
'{repo}::{hostname}-{timestamp}'.format(
|
'{repository}::{hostname}-{timestamp}'.format(
|
||||||
repo=repository,
|
repository=repository,
|
||||||
hostname=platform.node(),
|
hostname=platform.node(),
|
||||||
timestamp=datetime.now().isoformat(),
|
timestamp=datetime.now().isoformat(),
|
||||||
),
|
),
|
||||||
|
@ -104,7 +105,7 @@ def _make_prune_flags(retention_config):
|
||||||
def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None):
|
def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None):
|
||||||
'''
|
'''
|
||||||
Given a verbosity flag, a local or remote repository path, a retention config dict, and a
|
Given a verbosity flag, a local or remote repository path, a retention config dict, and a
|
||||||
command to run, prune attic archives according the the retention policy specified in that
|
command to run, prune Borg archives according the the retention policy specified in that
|
||||||
configuration.
|
configuration.
|
||||||
'''
|
'''
|
||||||
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
||||||
|
@ -170,33 +171,72 @@ def _make_check_flags(checks, check_last=None):
|
||||||
|
|
||||||
return tuple(
|
return tuple(
|
||||||
'--{}-only'.format(check) for check in checks
|
'--{}-only'.format(check) for check in checks
|
||||||
|
if check in DEFAULT_CHECKS
|
||||||
) + last_flag
|
) + last_flag
|
||||||
|
|
||||||
|
|
||||||
def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None):
|
def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None):
|
||||||
'''
|
'''
|
||||||
Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
|
Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
|
||||||
command to run, check the contained attic archives for consistency.
|
command to run, check the contained Borg archives for consistency.
|
||||||
|
|
||||||
If there are no consistency checks to run, skip running them.
|
If there are no consistency checks to run, skip running them.
|
||||||
'''
|
'''
|
||||||
checks = _parse_checks(consistency_config)
|
checks = _parse_checks(consistency_config)
|
||||||
check_last = consistency_config.get('check_last', None)
|
check_last = consistency_config.get('check_last', None)
|
||||||
if not checks:
|
|
||||||
return
|
|
||||||
|
|
||||||
|
if set(checks).intersection(set(DEFAULT_CHECKS)):
|
||||||
|
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
||||||
|
verbosity_flags = {
|
||||||
|
VERBOSITY_SOME: ('--info',),
|
||||||
|
VERBOSITY_LOTS: ('--debug',),
|
||||||
|
}.get(verbosity, ())
|
||||||
|
|
||||||
|
full_command = (
|
||||||
|
command, 'check',
|
||||||
|
repository,
|
||||||
|
) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
|
||||||
|
|
||||||
|
# The check command spews to stdout/stderr even without the verbose flag. Suppress it.
|
||||||
|
stdout = None if verbosity_flags else open(os.devnull, 'w')
|
||||||
|
|
||||||
|
subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
|
||||||
|
|
||||||
|
if 'extract' in checks:
|
||||||
|
extract_last_archive_dry_run(verbosity, repository, command, remote_path)
|
||||||
|
|
||||||
|
|
||||||
|
def extract_last_archive_dry_run(verbosity, repository, command=COMMAND, remote_path=None):
|
||||||
|
'''
|
||||||
|
Perform an extraction dry-run of just the most recent archive. If there are no archives, skip
|
||||||
|
the dry-run.
|
||||||
|
'''
|
||||||
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
|
||||||
verbosity_flags = {
|
verbosity_flags = {
|
||||||
VERBOSITY_SOME: ('--info',),
|
VERBOSITY_SOME: ('--info',),
|
||||||
VERBOSITY_LOTS: ('--debug',),
|
VERBOSITY_LOTS: ('--debug',),
|
||||||
}.get(verbosity, ())
|
}.get(verbosity, ())
|
||||||
|
|
||||||
full_command = (
|
full_list_command = (
|
||||||
command, 'check',
|
command, 'list',
|
||||||
|
'--short',
|
||||||
repository,
|
repository,
|
||||||
) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags
|
) + remote_path_flags + verbosity_flags
|
||||||
|
|
||||||
# The check command spews to stdout/stderr even without the verbose flag. Suppress it.
|
list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding)
|
||||||
stdout = None if verbosity_flags else open(os.devnull, 'w')
|
|
||||||
|
|
||||||
subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)
|
last_archive_name = list_output.strip().split('\n')[-1]
|
||||||
|
if not last_archive_name:
|
||||||
|
return
|
||||||
|
|
||||||
|
list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else ()
|
||||||
|
full_extract_command = (
|
||||||
|
command, 'extract',
|
||||||
|
'--dry-run',
|
||||||
|
'{repository}::{last_archive_name}'.format(
|
||||||
|
repository=repository,
|
||||||
|
last_archive_name=last_archive_name,
|
||||||
|
),
|
||||||
|
) + remote_path_flags + verbosity_flags + list_flag
|
||||||
|
|
||||||
|
subprocess.check_call(full_extract_command)
|
||||||
|
|
|
@ -106,21 +106,25 @@ map:
|
||||||
consistency:
|
consistency:
|
||||||
desc: |
|
desc: |
|
||||||
Consistency checks to run after backups. See
|
Consistency checks to run after backups. See
|
||||||
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
|
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and
|
||||||
|
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details.
|
||||||
map:
|
map:
|
||||||
checks:
|
checks:
|
||||||
seq:
|
seq:
|
||||||
- type: str
|
- type: str
|
||||||
enum: ['repository', 'archives', 'disabled']
|
enum: ['repository', 'archives', 'extract', 'disabled']
|
||||||
unique: true
|
unique: true
|
||||||
desc: |
|
desc: |
|
||||||
List of consistency checks to run: "repository", "archives", or both. Defaults
|
List of one or more consistency checks to run: "repository", "archives", and/or
|
||||||
to both. Set to "disabled" to disable all consistency checks. See
|
"extract". Defaults to "repository" and "archives". Set to "disabled" to disable
|
||||||
https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
|
all consistency checks. "repository" checks the consistency of the repository,
|
||||||
|
"archive" checks all of the archives, and "extract" does an extraction dry-run
|
||||||
|
of just the most recent archive.
|
||||||
example:
|
example:
|
||||||
- repository
|
- repository
|
||||||
- archives
|
- archives
|
||||||
check_last:
|
check_last:
|
||||||
type: int
|
type: int
|
||||||
desc: Restrict the number of checked archives to the last n.
|
desc: Restrict the number of checked archives to the last n. Applies only to the
|
||||||
|
"archives" check.
|
||||||
example: 3
|
example: 3
|
||||||
|
|
|
@ -4,6 +4,7 @@ import sys
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from flexmock import flexmock
|
from flexmock import flexmock
|
||||||
|
import pytest
|
||||||
|
|
||||||
from borgmatic import borg as module
|
from borgmatic import borg as module
|
||||||
from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
|
from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
|
||||||
|
@ -46,17 +47,23 @@ def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
|
||||||
|
|
||||||
|
|
||||||
def insert_subprocess_mock(check_call_command, **kwargs):
|
def insert_subprocess_mock(check_call_command, **kwargs):
|
||||||
subprocess = flexmock(STDOUT=STDOUT)
|
subprocess = flexmock(module.subprocess)
|
||||||
subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
|
subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
|
||||||
flexmock(module).subprocess = subprocess
|
flexmock(module).subprocess = subprocess
|
||||||
|
|
||||||
|
|
||||||
def insert_subprocess_never():
|
def insert_subprocess_never():
|
||||||
subprocess = flexmock()
|
subprocess = flexmock(module.subprocess)
|
||||||
subprocess.should_receive('check_call').never()
|
subprocess.should_receive('check_call').never()
|
||||||
flexmock(module).subprocess = subprocess
|
flexmock(module).subprocess = subprocess
|
||||||
|
|
||||||
|
|
||||||
|
def insert_subprocess_check_output_mock(check_output_command, result, **kwargs):
|
||||||
|
subprocess = flexmock(module.subprocess)
|
||||||
|
subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once()
|
||||||
|
flexmock(module).subprocess = subprocess
|
||||||
|
|
||||||
|
|
||||||
def insert_platform_mock():
|
def insert_platform_mock():
|
||||||
flexmock(module.platform).should_receive('node').and_return('host')
|
flexmock(module.platform).should_receive('node').and_return('host')
|
||||||
|
|
||||||
|
@ -395,9 +402,15 @@ def test_parse_checks_with_disabled_returns_no_checks():
|
||||||
|
|
||||||
|
|
||||||
def test_make_check_flags_with_checks_returns_flags():
|
def test_make_check_flags_with_checks_returns_flags():
|
||||||
flags = module._make_check_flags(('foo', 'bar'))
|
flags = module._make_check_flags(('repository',))
|
||||||
|
|
||||||
assert flags == ('--foo-only', '--bar-only')
|
assert flags == ('--repository-only',)
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_check_flags_with_extract_check_does_not_make_extract_flag():
|
||||||
|
flags = module._make_check_flags(('extract',))
|
||||||
|
|
||||||
|
assert flags == ()
|
||||||
|
|
||||||
|
|
||||||
def test_make_check_flags_with_default_checks_returns_no_flags():
|
def test_make_check_flags_with_default_checks_returns_no_flags():
|
||||||
|
@ -407,19 +420,27 @@ def test_make_check_flags_with_default_checks_returns_no_flags():
|
||||||
|
|
||||||
|
|
||||||
def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
|
def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
|
||||||
flags = module._make_check_flags(('foo', 'bar'), check_last=3)
|
flags = module._make_check_flags(('repository',), check_last=3)
|
||||||
|
|
||||||
assert flags == ('--foo-only', '--bar-only', '--last', '3')
|
assert flags == ('--repository-only', '--last', '3')
|
||||||
|
|
||||||
|
|
||||||
def test_make_check_flags_with_last_returns_last_flag():
|
def test_make_check_flags_with_default_checks_and_last_returns_last_flag():
|
||||||
flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
|
flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
|
||||||
|
|
||||||
assert flags == ('--last', '3')
|
assert flags == ('--last', '3')
|
||||||
|
|
||||||
|
|
||||||
def test_check_archives_should_call_borg_with_parameters():
|
@pytest.mark.parametrize(
|
||||||
checks = flexmock()
|
'checks',
|
||||||
|
(
|
||||||
|
('repository',),
|
||||||
|
('archives',),
|
||||||
|
('repository', 'archives'),
|
||||||
|
('repository', 'archives', 'other'),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
def test_check_archives_should_call_borg_with_parameters(checks):
|
||||||
check_last = flexmock()
|
check_last = flexmock()
|
||||||
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
|
@ -442,9 +463,27 @@ def test_check_archives_should_call_borg_with_parameters():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_archives_with_extract_check_should_call_extract_only():
|
||||||
|
checks = ('extract',)
|
||||||
|
check_last = flexmock()
|
||||||
|
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
||||||
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
|
flexmock(module).should_receive('_make_check_flags').never()
|
||||||
|
flexmock(module).should_receive('extract_last_archive_dry_run').once()
|
||||||
|
insert_subprocess_never()
|
||||||
|
|
||||||
|
module.check_archives(
|
||||||
|
verbosity=None,
|
||||||
|
repository='repo',
|
||||||
|
consistency_config=consistency_config,
|
||||||
|
command='borg',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter():
|
def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter():
|
||||||
|
checks = ('repository',)
|
||||||
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
flexmock(module).should_receive('_make_check_flags').and_return(())
|
flexmock(module).should_receive('_make_check_flags').and_return(())
|
||||||
insert_subprocess_mock(
|
insert_subprocess_mock(
|
||||||
('borg', 'check', 'repo', '--info'),
|
('borg', 'check', 'repo', '--info'),
|
||||||
|
@ -462,8 +501,9 @@ def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter
|
||||||
|
|
||||||
|
|
||||||
def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
|
def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter():
|
||||||
|
checks = ('repository',)
|
||||||
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
consistency_config = flexmock().should_receive('get').and_return(None).mock
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
flexmock(module).should_receive('_make_check_flags').and_return(())
|
flexmock(module).should_receive('_make_check_flags').and_return(())
|
||||||
insert_subprocess_mock(
|
insert_subprocess_mock(
|
||||||
('borg', 'check', 'repo', '--debug'),
|
('borg', 'check', 'repo', '--debug'),
|
||||||
|
@ -494,7 +534,7 @@ def test_check_archives_without_any_checks_should_bail():
|
||||||
|
|
||||||
|
|
||||||
def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
|
def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters():
|
||||||
checks = flexmock()
|
checks = ('repository',)
|
||||||
check_last = flexmock()
|
check_last = flexmock()
|
||||||
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
consistency_config = flexmock().should_receive('get').and_return(check_last).mock
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
|
@ -516,3 +556,87 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param
|
||||||
command='borg',
|
command='borg',
|
||||||
remote_path='borg1',
|
remote_path='borg1',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_last_archive_dry_run_should_call_borg_with_last_archive():
|
||||||
|
flexmock(sys.stdout).encoding = 'utf-8'
|
||||||
|
insert_subprocess_check_output_mock(
|
||||||
|
('borg', 'list', '--short', 'repo'),
|
||||||
|
result='archive1\narchive2\n'.encode('utf-8'),
|
||||||
|
)
|
||||||
|
insert_subprocess_mock(
|
||||||
|
('borg', 'extract', '--dry-run', 'repo::archive2'),
|
||||||
|
)
|
||||||
|
|
||||||
|
module.extract_last_archive_dry_run(
|
||||||
|
verbosity=None,
|
||||||
|
repository='repo',
|
||||||
|
command='borg',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_last_archive_dry_run_without_any_archives_should_bail():
|
||||||
|
flexmock(sys.stdout).encoding = 'utf-8'
|
||||||
|
insert_subprocess_check_output_mock(
|
||||||
|
('borg', 'list', '--short', 'repo'),
|
||||||
|
result='\n'.encode('utf-8'),
|
||||||
|
)
|
||||||
|
insert_subprocess_never()
|
||||||
|
|
||||||
|
module.extract_last_archive_dry_run(
|
||||||
|
verbosity=None,
|
||||||
|
repository='repo',
|
||||||
|
command='borg',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter():
|
||||||
|
flexmock(sys.stdout).encoding = 'utf-8'
|
||||||
|
insert_subprocess_check_output_mock(
|
||||||
|
('borg', 'list', '--short', 'repo', '--info'),
|
||||||
|
result='archive1\narchive2\n'.encode('utf-8'),
|
||||||
|
)
|
||||||
|
insert_subprocess_mock(
|
||||||
|
('borg', 'extract', '--dry-run', 'repo::archive2', '--info'),
|
||||||
|
)
|
||||||
|
|
||||||
|
module.extract_last_archive_dry_run(
|
||||||
|
verbosity=VERBOSITY_SOME,
|
||||||
|
repository='repo',
|
||||||
|
command='borg',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter():
|
||||||
|
flexmock(sys.stdout).encoding = 'utf-8'
|
||||||
|
insert_subprocess_check_output_mock(
|
||||||
|
('borg', 'list', '--short', 'repo', '--debug'),
|
||||||
|
result='archive1\narchive2\n'.encode('utf-8'),
|
||||||
|
)
|
||||||
|
insert_subprocess_mock(
|
||||||
|
('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'),
|
||||||
|
)
|
||||||
|
|
||||||
|
module.extract_last_archive_dry_run(
|
||||||
|
verbosity=VERBOSITY_LOTS,
|
||||||
|
repository='repo',
|
||||||
|
command='borg',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters():
|
||||||
|
flexmock(sys.stdout).encoding = 'utf-8'
|
||||||
|
insert_subprocess_check_output_mock(
|
||||||
|
('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'),
|
||||||
|
result='archive1\narchive2\n'.encode('utf-8'),
|
||||||
|
)
|
||||||
|
insert_subprocess_mock(
|
||||||
|
('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'),
|
||||||
|
)
|
||||||
|
|
||||||
|
module.extract_last_archive_dry_run(
|
||||||
|
verbosity=None,
|
||||||
|
repository='repo',
|
||||||
|
command='borg',
|
||||||
|
remote_path='borg1',
|
||||||
|
)
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -1,7 +1,7 @@
|
||||||
from setuptools import setup, find_packages
|
from setuptools import setup, find_packages
|
||||||
|
|
||||||
|
|
||||||
VERSION = '1.1.4'
|
VERSION = '1.1.5'
|
||||||
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
|
|
Loading…
Reference in a new issue