Fix conflict between "patterns" and "source_directories" (#574), make "source_directories" optional (#542).

This commit is contained in:
Dan Helfman 2022-08-25 11:55:34 -07:00
parent a274c0dbf7
commit c46f2b8508
6 changed files with 39 additions and 18 deletions

9
NEWS
View file

@ -1,3 +1,12 @@
1.7.1.dev0
* #542: Make the "source_directories" option optional. This is useful for "check"-only setups or
using "patterns" exclusively.
* #574: Fix for potential data loss (data not getting backed up) when the "patterns" option was
used with "source_directories" (or the "~/.borgmatic" path existed, which got injected into
"source_directories" implicitly). The fix is for borgmatic to convert "source_directories" into
patterns whenever "patterns" is used, working around a potential Borg bug:
https://github.com/borgbackup/borg/issues/6994
1.7.0 1.7.0
* #463: Add "before_actions" and "after_actions" command hooks that run before/after all the * #463: Add "before_actions" and "after_actions" command hooks that run before/after all the
actions for each repository. These new hooks are a good place to run per-repository steps like actions for each repository. These new hooks are a good place to run per-repository steps like

View file

@ -98,16 +98,19 @@ def deduplicate_directories(directory_devices):
return tuple(sorted(deduplicated)) return tuple(sorted(deduplicated))
def write_pattern_file(patterns=None): def write_pattern_file(patterns=None, sources=None):
''' '''
Given a sequence of patterns, write them to a named temporary file and return it. Return None Given a sequence of patterns and an optional sequence of source directories, write them to a
if no patterns are provided. named temporary file (with the source directories as additional roots) and return the file.
Return None if no patterns are provided.
''' '''
if not patterns: if not patterns:
return None return None
pattern_file = tempfile.NamedTemporaryFile('w') pattern_file = tempfile.NamedTemporaryFile('w')
pattern_file.write('\n'.join(patterns)) pattern_file.write(
'\n'.join(tuple(patterns) + tuple(f'R {source}' for source in (sources or [])))
)
pattern_file.flush() pattern_file.flush()
return pattern_file return pattern_file
@ -216,7 +219,7 @@ def create_archive(
sources = deduplicate_directories( sources = deduplicate_directories(
map_directories_to_devices( map_directories_to_devices(
expand_directories( expand_directories(
location_config['source_directories'] location_config.get('source_directories', [])
+ borgmatic_source_directories(location_config.get('borgmatic_source_directory')) + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
) )
) )
@ -226,7 +229,7 @@ def create_archive(
working_directory = os.path.expanduser(location_config.get('working_directory')) working_directory = os.path.expanduser(location_config.get('working_directory'))
except TypeError: except TypeError:
working_directory = None working_directory = None
pattern_file = write_pattern_file(location_config.get('patterns')) pattern_file = write_pattern_file(location_config.get('patterns'), sources)
exclude_file = write_pattern_file( exclude_file = write_pattern_file(
expand_home_directories(location_config.get('exclude_patterns')) expand_home_directories(location_config.get('exclude_patterns'))
) )
@ -299,7 +302,7 @@ def create_archive(
+ (('--json',) if json else ()) + (('--json',) if json else ())
+ (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
+ flags.make_repository_archive_flags(repository, archive_name_format, local_borg_version) + flags.make_repository_archive_flags(repository, archive_name_format, local_borg_version)
+ sources + (sources if not pattern_file else ())
) )
if json: if json:

View file

@ -11,7 +11,6 @@ properties:
https://borgbackup.readthedocs.io/en/stable/usage/create.html https://borgbackup.readthedocs.io/en/stable/usage/create.html
for details. for details.
required: required:
- source_directories
- repositories - repositories
additionalProperties: false additionalProperties: false
properties: properties:
@ -20,8 +19,8 @@ properties:
items: items:
type: string type: string
description: | description: |
List of source directories to backup (required). Globs and List of source directories to backup. Globs and tildes are
tildes are expanded. Do not backslash spaces in path names. expanded. Do not backslash spaces in path names.
example: example:
- /home - /home
- /etc - /etc
@ -123,7 +122,8 @@ properties:
backups. Globs are expanded. (Tildes are not.) See the backups. Globs are expanded. (Tildes are not.) See the
output of "borg help patterns" for more details. Quote any output of "borg help patterns" for more details. Quote any
value if it contains leading punctuation, so it parses value if it contains leading punctuation, so it parses
correctly. correctly. Note that only one of "patterns" and
"source_directories" may be used.
example: example:
- 'R /' - 'R /'
- '- /home/*/.cache' - '- /home/*/.cache'

View file

@ -72,7 +72,7 @@ def apply_logical_validation(config_filename, parsed_configuration):
raise Validation_error( raise Validation_error(
config_filename, config_filename,
( (
'Unknown repository in the consistency section\'s check_repositories: {}'.format( 'Unknown repository in the "consistency" section\'s "check_repositories": {}'.format(
repository repository
), ),
), ),

View file

@ -1,6 +1,6 @@
from setuptools import find_packages, setup from setuptools import find_packages, setup
VERSION = '1.7.0' VERSION = '1.7.1.dev0'
setup( setup(

View file

@ -109,11 +109,20 @@ def test_deduplicate_directories_removes_child_paths_on_the_same_filesystem(
assert module.deduplicate_directories(directories) == expected_directories assert module.deduplicate_directories(directories) == expected_directories
def test_write_pattern_file_does_not_raise(): def test_write_pattern_file_writes_pattern_lines():
temporary_file = flexmock(name='filename', write=lambda mode: None, flush=lambda: None) temporary_file = flexmock(name='filename', flush=lambda: None)
temporary_file.should_receive('write').with_args('R /foo\n+ /foo/bar')
flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file) flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
module.write_pattern_file(['exclude']) module.write_pattern_file(['R /foo', '+ /foo/bar'])
def test_write_pattern_file_with_sources_writes_sources_as_roots():
temporary_file = flexmock(name='filename', flush=lambda: None)
temporary_file.should_receive('write').with_args('R /foo\n+ /foo/bar\nR /baz\nR /quux')
flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
module.write_pattern_file(['R /foo', '+ /foo/bar'], sources=['/baz', '/quux'])
def test_write_pattern_file_with_empty_exclude_patterns_does_not_raise(): def test_write_pattern_file_with_empty_exclude_patterns_does_not_raise():
@ -357,7 +366,7 @@ def test_create_archive_calls_borg_with_environment():
) )
def test_create_archive_with_patterns_calls_borg_with_patterns(): def test_create_archive_with_patterns_calls_borg_with_patterns_including_converted_source_directories():
pattern_flags = ('--patterns-from', 'patterns') pattern_flags = ('--patterns-from', 'patterns')
flexmock(module).should_receive('borgmatic_source_directories').and_return([]) flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
@ -377,7 +386,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
) )
flexmock(module.environment).should_receive('make_environment') flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('execute_command').with_args( flexmock(module).should_receive('execute_command').with_args(
('borg', 'create') + pattern_flags + REPO_ARCHIVE_WITH_PATHS, ('borg', 'create') + pattern_flags + (f'repo::{DEFAULT_ARCHIVE_NAME}',),
output_log_level=logging.INFO, output_log_level=logging.INFO,
output_file=None, output_file=None,
borg_local_path='borg', borg_local_path='borg',