Merge branch 'master' of ssh://projects.torsion.org:3022/witten/borgmatic
This commit is contained in:
commit
3930e63320
18 changed files with 77 additions and 30 deletions
13
NEWS
13
NEWS
|
@ -1,4 +1,15 @@
|
||||||
1.2.1.dev0
|
1.2.3.dev0
|
||||||
|
* #87: Support for Borg create --checkpoint-interval via "checkpoint_interval" option in
|
||||||
|
borgmatic's storage configuration.
|
||||||
|
* #88: Fix declared pykwalify compatibility version range in setup.py to prevent use of ancient
|
||||||
|
versions of pykwalify with large version numbers.
|
||||||
|
* #89: Pass --show-rc option to Borg when at highest verbosity level.
|
||||||
|
|
||||||
|
1.2.2
|
||||||
|
* #85: Fix compatibility issue between pykwalify and ruamel.yaml 0.15.52, which manifested in
|
||||||
|
borgmatic as a pykwalify RuleError.
|
||||||
|
|
||||||
|
1.2.1
|
||||||
* Skip before/after backup hooks when only doing --prune, --check, --list, and/or --info.
|
* Skip before/after backup hooks when only doing --prune, --check, --list, and/or --info.
|
||||||
* #71: Support for XDG_CONFIG_HOME environment variable for specifying alternate user ~/.config/
|
* #71: Support for XDG_CONFIG_HOME environment variable for specifying alternate user ~/.config/
|
||||||
path.
|
path.
|
||||||
|
|
|
@ -93,7 +93,7 @@ def check_archives(verbosity, repository, storage_config, consistency_config, lo
|
||||||
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
|
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
|
||||||
verbosity_flags = {
|
verbosity_flags = {
|
||||||
VERBOSITY_SOME: ('--info',),
|
VERBOSITY_SOME: ('--info',),
|
||||||
VERBOSITY_LOTS: ('--debug',),
|
VERBOSITY_LOTS: ('--debug', '--show-rc'),
|
||||||
}.get(verbosity, ())
|
}.get(verbosity, ())
|
||||||
prefix = consistency_config.get('prefix')
|
prefix = consistency_config.get('prefix')
|
||||||
|
|
||||||
|
|
|
@ -115,6 +115,7 @@ def create_archive(
|
||||||
|
|
||||||
pattern_file = _write_pattern_file(location_config.get('patterns'))
|
pattern_file = _write_pattern_file(location_config.get('patterns'))
|
||||||
exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns')))
|
exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns')))
|
||||||
|
checkpoint_interval = storage_config.get('checkpoint_interval', None)
|
||||||
compression = storage_config.get('compression', None)
|
compression = storage_config.get('compression', None)
|
||||||
remote_rate_limit = storage_config.get('remote_rate_limit', None)
|
remote_rate_limit = storage_config.get('remote_rate_limit', None)
|
||||||
umask = storage_config.get('umask', None)
|
umask = storage_config.get('umask', None)
|
||||||
|
@ -140,6 +141,7 @@ def create_archive(
|
||||||
location_config,
|
location_config,
|
||||||
exclude_file.name if exclude_file else None,
|
exclude_file.name if exclude_file else None,
|
||||||
)
|
)
|
||||||
|
+ (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
|
||||||
+ (('--compression', compression) if compression else ())
|
+ (('--compression', compression) if compression else ())
|
||||||
+ (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
|
+ (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
|
||||||
+ (('--one-file-system',) if location_config.get('one_file_system') else ())
|
+ (('--one-file-system',) if location_config.get('one_file_system') else ())
|
||||||
|
@ -149,8 +151,8 @@ def create_archive(
|
||||||
+ (('--umask', str(umask)) if umask else ())
|
+ (('--umask', str(umask)) if umask else ())
|
||||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||||
+ {
|
+ {
|
||||||
VERBOSITY_SOME: ('--info',) if dry_run else ('--info', '--stats',),
|
VERBOSITY_SOME: ('--info',) if dry_run else ('--info', '--stats'),
|
||||||
VERBOSITY_LOTS: ('--debug', '--list',) if dry_run else ('--debug', '--list', '--stats',),
|
VERBOSITY_LOTS: ('--debug', '--list', '--show-rc') if dry_run else ('--debug', '--list', '--show-rc', '--stats'),
|
||||||
}.get(verbosity, ())
|
}.get(verbosity, ())
|
||||||
+ (('--dry-run',) if dry_run else ())
|
+ (('--dry-run',) if dry_run else ())
|
||||||
)
|
)
|
||||||
|
|
|
@ -17,7 +17,7 @@ def extract_last_archive_dry_run(verbosity, repository, lock_wait=None, local_pa
|
||||||
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
|
lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
|
||||||
verbosity_flags = {
|
verbosity_flags = {
|
||||||
VERBOSITY_SOME: ('--info',),
|
VERBOSITY_SOME: ('--info',),
|
||||||
VERBOSITY_LOTS: ('--debug',),
|
VERBOSITY_LOTS: ('--debug', '--show-rc'),
|
||||||
}.get(verbosity, ())
|
}.get(verbosity, ())
|
||||||
|
|
||||||
full_list_command = (
|
full_list_command = (
|
||||||
|
|
|
@ -23,7 +23,7 @@ def display_archives_info(
|
||||||
+ (('--json',) if json else ())
|
+ (('--json',) if json else ())
|
||||||
+ {
|
+ {
|
||||||
VERBOSITY_SOME: ('--info',),
|
VERBOSITY_SOME: ('--info',),
|
||||||
VERBOSITY_LOTS: ('--debug',),
|
VERBOSITY_LOTS: ('--debug', '--show-rc'),
|
||||||
}.get(verbosity, ())
|
}.get(verbosity, ())
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -23,7 +23,7 @@ def list_archives(
|
||||||
+ (('--json',) if json else ())
|
+ (('--json',) if json else ())
|
||||||
+ {
|
+ {
|
||||||
VERBOSITY_SOME: ('--info',),
|
VERBOSITY_SOME: ('--info',),
|
||||||
VERBOSITY_LOTS: ('--debug',),
|
VERBOSITY_LOTS: ('--debug', '--show-rc'),
|
||||||
}.get(verbosity, ())
|
}.get(verbosity, ())
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -56,7 +56,7 @@ def prune_archives(verbosity, dry_run, repository, storage_config, retention_con
|
||||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||||
+ {
|
+ {
|
||||||
VERBOSITY_SOME: ('--info', '--stats',),
|
VERBOSITY_SOME: ('--info', '--stats',),
|
||||||
VERBOSITY_LOTS: ('--debug', '--stats', '--list'),
|
VERBOSITY_LOTS: ('--debug', '--stats', '--list', '--show-rc'),
|
||||||
}.get(verbosity, ())
|
}.get(verbosity, ())
|
||||||
+ (('--dry-run',) if dry_run else ())
|
+ (('--dry-run',) if dry_run else ())
|
||||||
)
|
)
|
||||||
|
|
|
@ -81,7 +81,7 @@ def parse_arguments(*arguments):
|
||||||
dest='json',
|
dest='json',
|
||||||
default=False,
|
default=False,
|
||||||
action='store_true',
|
action='store_true',
|
||||||
help='Output results from the --list option as json',
|
help='Output results from the --list or --info options as json',
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-n', '--dry-run',
|
'-n', '--dry-run',
|
||||||
|
|
|
@ -124,6 +124,13 @@ map:
|
||||||
punctuation, so it parses correctly. And backslash any quote or backslash
|
punctuation, so it parses correctly. And backslash any quote or backslash
|
||||||
literals as well.
|
literals as well.
|
||||||
example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
|
example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
|
||||||
|
checkpoint_interval:
|
||||||
|
type: int
|
||||||
|
desc: |
|
||||||
|
Number of seconds between each checkpoint during a long-running backup. See
|
||||||
|
https://borgbackup.readthedocs.io/en/stable/faq.html#if-a-backup-stops-mid-way-does-the-already-backed-up-data-stay-there
|
||||||
|
for details. Defaults to checkpoints every 1800 seconds (30 minutes).
|
||||||
|
example: 1800
|
||||||
compression:
|
compression:
|
||||||
type: scalar
|
type: scalar
|
||||||
desc: |
|
desc: |
|
||||||
|
|
|
@ -74,8 +74,8 @@ def parse_configuration(config_filename, schema_filename):
|
||||||
logging.getLogger('pykwalify').setLevel(logging.ERROR)
|
logging.getLogger('pykwalify').setLevel(logging.ERROR)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
config = yaml.round_trip_load(open(config_filename))
|
config = yaml.safe_load(open(config_filename))
|
||||||
schema = yaml.round_trip_load(open(schema_filename))
|
schema = yaml.safe_load(open(schema_filename))
|
||||||
except yaml.error.YAMLError as error:
|
except yaml.error.YAMLError as error:
|
||||||
raise Validation_error(config_filename, (str(error),))
|
raise Validation_error(config_filename, (str(error),))
|
||||||
|
|
||||||
|
|
|
@ -173,7 +173,7 @@ def test_check_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
|
||||||
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
flexmock(module).should_receive('_parse_checks').and_return(checks)
|
||||||
flexmock(module).should_receive('_make_check_flags').and_return(())
|
flexmock(module).should_receive('_make_check_flags').and_return(())
|
||||||
insert_subprocess_mock(
|
insert_subprocess_mock(
|
||||||
('borg', 'check', 'repo', '--debug'),
|
('borg', 'check', 'repo', '--debug', '--show-rc'),
|
||||||
stdout=None, stderr=STDOUT,
|
stdout=None, stderr=STDOUT,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -296,7 +296,7 @@ def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
|
||||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||||
insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
|
insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--show-rc', '--stats'))
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
verbosity=VERBOSITY_LOTS,
|
verbosity=VERBOSITY_LOTS,
|
||||||
|
@ -359,7 +359,7 @@ def test_create_archive_with_dry_run_and_verbosity_lots_calls_borg_without_stats
|
||||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||||
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||||
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||||
insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--dry-run'))
|
insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--show-rc', '--dry-run'))
|
||||||
|
|
||||||
module.create_archive(
|
module.create_archive(
|
||||||
verbosity=VERBOSITY_LOTS,
|
verbosity=VERBOSITY_LOTS,
|
||||||
|
@ -374,6 +374,26 @@ def test_create_archive_with_dry_run_and_verbosity_lots_calls_borg_without_stats
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
|
||||||
|
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
|
||||||
|
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||||
|
flexmock(module).should_receive('_make_pattern_flags').and_return(())
|
||||||
|
flexmock(module).should_receive('_make_exclude_flags').and_return(())
|
||||||
|
insert_subprocess_mock(CREATE_COMMAND + ('--checkpoint-interval', '600'))
|
||||||
|
|
||||||
|
module.create_archive(
|
||||||
|
verbosity=None,
|
||||||
|
dry_run=False,
|
||||||
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={'checkpoint_interval': 600},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_create_archive_with_compression_calls_borg_with_compression_parameters():
|
def test_create_archive_with_compression_calls_borg_with_compression_parameters():
|
||||||
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
|
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
|
||||||
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
flexmock(module).should_receive('_write_pattern_file').and_return(None)
|
||||||
|
|
|
@ -73,11 +73,11 @@ def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_
|
||||||
def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter():
|
def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter():
|
||||||
flexmock(sys.stdout).encoding = 'utf-8'
|
flexmock(sys.stdout).encoding = 'utf-8'
|
||||||
insert_subprocess_check_output_mock(
|
insert_subprocess_check_output_mock(
|
||||||
('borg', 'list', '--short', 'repo', '--debug'),
|
('borg', 'list', '--short', 'repo', '--debug', '--show-rc'),
|
||||||
result='archive1\narchive2\n'.encode('utf-8'),
|
result='archive1\narchive2\n'.encode('utf-8'),
|
||||||
)
|
)
|
||||||
insert_subprocess_mock(
|
insert_subprocess_mock(
|
||||||
('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'),
|
('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--show-rc', '--list'),
|
||||||
)
|
)
|
||||||
|
|
||||||
module.extract_last_archive_dry_run(
|
module.extract_last_archive_dry_run(
|
||||||
|
|
|
@ -35,7 +35,7 @@ def test_display_archives_info_with_verbosity_some_calls_borg_with_info_paramete
|
||||||
|
|
||||||
|
|
||||||
def test_display_archives_info_with_verbosity_lots_calls_borg_with_debug_parameter():
|
def test_display_archives_info_with_verbosity_lots_calls_borg_with_debug_parameter():
|
||||||
insert_subprocess_mock(INFO_COMMAND + ('--debug',))
|
insert_subprocess_mock(INFO_COMMAND + ('--debug', '--show-rc'))
|
||||||
|
|
||||||
module.display_archives_info(
|
module.display_archives_info(
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
|
|
@ -35,7 +35,7 @@ def test_list_archives_with_verbosity_some_calls_borg_with_info_parameter():
|
||||||
|
|
||||||
|
|
||||||
def test_list_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
|
def test_list_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
|
||||||
insert_subprocess_mock(LIST_COMMAND + ('--debug',))
|
insert_subprocess_mock(LIST_COMMAND + ('--debug', '--show-rc'))
|
||||||
|
|
||||||
module.list_archives(
|
module.list_archives(
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
|
|
@ -92,7 +92,7 @@ def test_prune_archives_with_verbosity_lots_calls_borg_with_debug_parameter():
|
||||||
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
|
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
|
||||||
BASE_PRUNE_FLAGS,
|
BASE_PRUNE_FLAGS,
|
||||||
)
|
)
|
||||||
insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats', '--list'))
|
insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats', '--list', '--show-rc'))
|
||||||
|
|
||||||
module.prune_archives(
|
module.prune_archives(
|
||||||
repository='repo',
|
repository='repo',
|
||||||
|
|
|
@ -22,22 +22,29 @@ for sub_command in prune create check list info; do
|
||||||
sort borgmatic_borg_flags > borgmatic_borg_flags.sorted
|
sort borgmatic_borg_flags > borgmatic_borg_flags.sorted
|
||||||
mv borgmatic_borg_flags.sorted borgmatic_borg_flags
|
mv borgmatic_borg_flags.sorted borgmatic_borg_flags
|
||||||
|
|
||||||
for line in $(borg $sub_command --help | awk -v RS= '/^usage:/') ; do
|
for word in $(borg $sub_command --help | grep '^ -') ; do
|
||||||
# Exclude a bunch of flags that borgmatic actually supports, but don't get exercised by the
|
# Exclude a bunch of flags that borgmatic actually supports, but don't get exercised by the
|
||||||
# generated sample config, and also flags that don't make sense to support.
|
# generated sample config, and also flags that don't make sense to support.
|
||||||
echo "$line" | grep -- -- | sed -r 's/(\[|\])//g' \
|
echo "$word" | grep ^-- | sed -e 's/,$//' \
|
||||||
| grep -v '^-h$' \
|
|
||||||
| grep -v '^--archives-only$' \
|
| grep -v '^--archives-only$' \
|
||||||
| grep -v '^--repository-only$' \
|
|
||||||
| grep -v '^--stats$' \
|
|
||||||
| grep -v '^--list$' \
|
|
||||||
| grep -v '^--critical$' \
|
| grep -v '^--critical$' \
|
||||||
| grep -v '^--error$' \
|
|
||||||
| grep -v '^--warning$' \
|
|
||||||
| grep -v '^--info$' \
|
|
||||||
| grep -v '^--debug$' \
|
| grep -v '^--debug$' \
|
||||||
|
| grep -v '^--dry-run$' \
|
||||||
|
| grep -v '^--error$' \
|
||||||
|
| grep -v '^--help$' \
|
||||||
|
| grep -v '^--info$' \
|
||||||
|
| grep -v '^--list$' \
|
||||||
|
| grep -v '^--nobsdflags$' \
|
||||||
|
| grep -v '^--pattern$' \
|
||||||
|
| grep -v '^--repository-only$' \
|
||||||
|
| grep -v '^--show-rc$' \
|
||||||
|
| grep -v '^--stats$' \
|
||||||
|
| grep -v '^--verbose$' \
|
||||||
|
| grep -v '^--warning$' \
|
||||||
|
| grep -v '^-h$' \
|
||||||
>> all_borg_flags
|
>> all_borg_flags
|
||||||
done
|
done
|
||||||
|
|
||||||
sort all_borg_flags > all_borg_flags.sorted
|
sort all_borg_flags > all_borg_flags.sorted
|
||||||
mv all_borg_flags.sorted all_borg_flags
|
mv all_borg_flags.sorted all_borg_flags
|
||||||
|
|
||||||
|
|
4
setup.py
4
setup.py
|
@ -1,7 +1,7 @@
|
||||||
from setuptools import setup, find_packages
|
from setuptools import setup, find_packages
|
||||||
|
|
||||||
|
|
||||||
VERSION = '1.2.1.dev0'
|
VERSION = '1.2.3.dev0'
|
||||||
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
|
@ -32,7 +32,7 @@ setup(
|
||||||
'atticmatic',
|
'atticmatic',
|
||||||
],
|
],
|
||||||
install_requires=(
|
install_requires=(
|
||||||
'pykwalify>=1.6.0',
|
'pykwalify>=1.6.0,<14.06',
|
||||||
'ruamel.yaml>0.15.0,<0.16.0',
|
'ruamel.yaml>0.15.0,<0.16.0',
|
||||||
'setuptools',
|
'setuptools',
|
||||||
),
|
),
|
||||||
|
|
Loading…
Reference in a new issue