49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed includes/excludes.

This commit is contained in:
Dan Helfman 2018-01-14 15:52:19 -08:00
parent 50b3240c4f
commit b8f6bab12d
4 changed files with 152 additions and 41 deletions

4
NEWS
View file

@ -1,6 +1,8 @@
1.1.13.dev0
* #54: Fix for incorrect consistency check flags passed to Borg when all three checks ("repository",
"archives", and "extract") are specified in borgmatic configuration.
* #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed
includes/excludes.
* Moved issue tracker from Taiga to integrated Gitea tracker at
https://projects.torsion.org/witten/borgmatic/issues
@ -19,7 +21,7 @@
shuts down if borgmatic is terminated (e.g. due to a system suspend).
* #30: Support for using tilde in repository paths to reference home directory.
* #43: Support for Borg --files-cache option for setting the files cache operation mode.
* #45: Support for Borg --remote-ratelimit for limiting upload rate.
* #45: Support for Borg --remote-ratelimit option for limiting upload rate.
* Log invoked Borg commands when at highest verbosity level.
1.1.9

View file

@ -31,28 +31,45 @@ def _expand_directory(directory):
return glob.glob(expanded_directory) or [expanded_directory]
def _write_exclude_file(exclude_patterns=None):
def _write_pattern_file(patterns=None):
'''
Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
None if no patterns are provided.
Given a sequence of patterns, write them to a named temporary file and return it. Return None
if no patterns are provided.
'''
if not exclude_patterns:
if not patterns:
return None
exclude_file = tempfile.NamedTemporaryFile('w')
exclude_file.write('\n'.join(exclude_patterns))
exclude_file.flush()
pattern_file = tempfile.NamedTemporaryFile('w')
pattern_file.write('\n'.join(patterns))
pattern_file.flush()
return exclude_file
return pattern_file
def _make_exclude_flags(location_config, exclude_patterns_filename=None):
def _make_pattern_flags(location_config, pattern_filename=None):
'''
Given a location config dict with a potential pattern_from option, and a filename containing any
additional patterns, return the corresponding Borg flags for those files as a tuple.
'''
pattern_filenames = tuple(location_config.get('patterns_from') or ()) + (
(pattern_filename,) if pattern_filename else ()
)
return tuple(
itertools.chain.from_iterable(
('--pattern-from', pattern_filename)
for pattern_filename in pattern_filenames
)
)
def _make_exclude_flags(location_config, exclude_filename=None):
'''
Given a location config dict with various exclude options, and a filename containing any exclude
patterns, return the corresponding Borg flags as a tuple.
'''
exclude_filenames = tuple(location_config.get('exclude_from') or ()) + (
(exclude_patterns_filename,) if exclude_patterns_filename else ()
(exclude_filename,) if exclude_filename else ()
)
exclude_from_flags = tuple(
itertools.chain.from_iterable(
@ -81,10 +98,15 @@ def create_archive(
)
)
exclude_patterns_file = _write_exclude_file(location_config.get('exclude_patterns'))
pattern_file = _write_pattern_file(location_config.get('patterns'))
pattern_flags = _make_pattern_flags(
location_config,
pattern_file.name if pattern_file else None,
)
exclude_file = _write_pattern_file(location_config.get('exclude_patterns'))
exclude_flags = _make_exclude_flags(
location_config,
exclude_patterns_file.name if exclude_patterns_file else None,
exclude_file.name if exclude_file else None,
)
compression = storage_config.get('compression', None)
compression_flags = ('--compression', compression) if compression else ()
@ -110,7 +132,7 @@ def create_archive(
repository=repository,
archive_name_format=archive_name_format,
),
) + sources + exclude_flags + compression_flags + remote_rate_limit_flags + \
) + sources + pattern_flags + exclude_flags + compression_flags + remote_rate_limit_flags + \
one_file_system_flags + files_cache_flags + remote_path_flags + umask_flags + \
verbosity_flags

View file

@ -42,6 +42,28 @@ map:
repositories are backed up to in sequence.
example:
- user@backupserver:sourcehostname.borg
patterns:
seq:
- type: scalar
desc: |
Any paths matching these patterns are included/excluded from backups. Globs are
expanded. Note that Borg considers this option experimental. See the output of
"borg help patterns" for more details. Quoting any value if it contains leading
punctuation, so it parses correctly.
example:
- 'R /'
- '- /home/*/.cache'
- '+ /home/susan'
- '- /home/*'
patterns_from:
seq:
- type: scalar
desc: |
Read include/exclude patterns from one or more separate named files, one pattern
per line. Note that Borg considers this option experimental. See the output of
"borg help patterns" for more details.
example:
- /etc/borgmatic/patterns
exclude_patterns:
seq:
- type: scalar
@ -57,7 +79,7 @@ map:
- type: scalar
desc: |
Read exclude patterns from one or more separate named files, one pattern per
line.
line. See the output of "borg help patterns" for more details.
example:
- /etc/borgmatic/excludes
exclude_caches:

View file

@ -58,7 +58,7 @@ def test_expand_directory_with_glob_expands():
assert paths == ['foo', 'food']
def test_write_exclude_file_does_not_raise():
def test_write_pattern_file_does_not_raise():
temporary_file = flexmock(
name='filename',
write=lambda mode: None,
@ -66,11 +66,11 @@ def test_write_exclude_file_does_not_raise():
)
flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
module._write_exclude_file(['exclude'])
module._write_pattern_file(['exclude'])
def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
module._write_exclude_file([])
def test_write_pattern_file_with_empty_exclude_patterns_does_not_raise():
module._write_pattern_file([])
def insert_subprocess_mock(check_call_command, **kwargs):
@ -78,17 +78,50 @@ def insert_subprocess_mock(check_call_command, **kwargs):
subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
def test_make_pattern_flags_includes_pattern_filename_when_given():
pattern_flags = module._make_pattern_flags(
location_config={'patterns': ['R /', '- /var']},
pattern_filename='/tmp/patterns',
)
assert pattern_flags == ('--pattern-from', '/tmp/patterns')
def test_make_pattern_flags_includes_patterns_from_filenames_when_in_config():
pattern_flags = module._make_pattern_flags(
location_config={'patterns_from': ['patterns', 'other']},
)
assert pattern_flags == ('--pattern-from', 'patterns', '--pattern-from', 'other')
def test_make_pattern_flags_includes_both_filenames_when_patterns_given_and_patterns_from_in_config():
pattern_flags = module._make_pattern_flags(
location_config={'patterns_from': ['patterns']},
pattern_filename='/tmp/patterns',
)
assert pattern_flags == ('--pattern-from', 'patterns', '--pattern-from', '/tmp/patterns')
def test_make_pattern_flags_considers_none_patterns_from_filenames_as_empty():
pattern_flags = module._make_pattern_flags(
location_config={'patterns_from': None},
)
assert pattern_flags == ()
def test_make_exclude_flags_includes_exclude_patterns_filename_when_given():
exclude_flags = module._make_exclude_flags(
location_config={'exclude_patterns': ['*.pyc', '/var']},
exclude_patterns_filename='/tmp/excludes',
exclude_filename='/tmp/excludes',
)
assert exclude_flags == ('--exclude-from', '/tmp/excludes')
def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
flexmock(module).should_receive('_write_exclude_file').and_return(None)
exclude_flags = module._make_exclude_flags(
location_config={'exclude_from': ['excludes', 'other']},
@ -98,19 +131,15 @@ def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config():
def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config():
flexmock(module).should_receive('_write_exclude_file').and_return(None)
exclude_flags = module._make_exclude_flags(
location_config={'exclude_from': ['excludes']},
exclude_patterns_filename='/tmp/excludes',
exclude_filename='/tmp/excludes',
)
assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes')
def test_make_exclude_flags_considers_none_exclude_from_filenames_as_empty():
flexmock(module).should_receive('_write_exclude_file').and_return(None)
exclude_flags = module._make_exclude_flags(
location_config={'exclude_from': None},
)
@ -154,7 +183,8 @@ CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'fo
def test_create_archive_calls_borg_with_parameters():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND)
@ -170,10 +200,31 @@ def test_create_archive_calls_borg_with_parameters():
)
def test_create_archive_with_patterns_calls_borg_with_patterns():
pattern_flags = ('--patterns-from', 'patterns')
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_pattern_file').and_return(flexmock(name='/tmp/patterns')).and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags)
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + pattern_flags)
module.create_archive(
verbosity=None,
repository='repo',
location_config={
'source_directories': ['foo', 'bar'],
'repositories': ['repo'],
'patterns': ['pattern'],
},
storage_config={},
)
def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
exclude_flags = ('--exclude-from', 'excludes')
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes'))
flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(flexmock(name='/tmp/excludes'))
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
@ -191,7 +242,9 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
@ -209,7 +262,8 @@ def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
@ -227,7 +281,8 @@ def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
def test_create_archive_with_compression_calls_borg_with_compression_parameters():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
@ -245,7 +300,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + ('--remote-ratelimit', '100'))
@ -263,7 +319,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
@ -282,7 +339,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + ('--files-cache', 'ctime,size'))
@ -301,7 +359,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
@ -320,7 +379,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
def test_create_archive_with_umask_calls_borg_with_umask_parameters():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
@ -338,7 +398,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
def test_create_archive_with_source_directories_glob_expands():
flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
@ -357,7 +418,8 @@ def test_create_archive_with_source_directories_glob_expands():
def test_create_archive_with_non_matching_source_directories_glob_passes_through():
flexmock(module).should_receive('_expand_directory').and_return(['foo*'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'))
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
@ -376,7 +438,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
def test_create_archive_with_glob_calls_borg_with_expanded_directories():
flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'))
@ -394,7 +457,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'))
@ -414,7 +478,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
flexmock(module).should_receive('_write_exclude_file').and_return(None)
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
flexmock(module).should_receive('_make_exclude_flags').and_return(())
insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'))