From 96df52ec5042d16515616ba4a5f4a5d4254f9466 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 20 May 2020 13:33:53 -0700 Subject: [PATCH] Fix hang when streaming a database dump to Borg with implicit duplicate source directories by deduplicating them first (#316). --- NEWS | 2 + borgmatic/borg/create.py | 38 ++++++++- docs/how-to/set-up-backups.md | 2 +- tests/unit/borg/test_create.py | 137 +++++++++++++++++++++++---------- 4 files changed, 136 insertions(+), 43 deletions(-) diff --git a/NEWS b/NEWS index b93bc8c..79b225b 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ 1.5.5.dev0 * #314: Fix regression in support for PostgreSQL's "directory" dump format. Unlike other dump formats, the "directory" dump format does not stream directly to/from Borg. + * #316: Fix hang when streaming a database dump to Borg with implicit duplicate source directories + by deduplicating them first. * Improve documentation around the installation process. Specifically, making borgmatic commands runnable via the system PATH and offering a global install option. diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 8ea2baf..5cb720f 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -2,6 +2,7 @@ import glob import itertools import logging import os +import pathlib import tempfile from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_with_processes @@ -43,6 +44,35 @@ def _expand_home_directories(directories): return tuple(os.path.expanduser(directory) for directory in directories) +def deduplicate_directories(directories): + ''' + Given a sequence of directories, return them as a sorted tuple with all duplicate child + directories removed. For instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',) + + The idea is that if Borg is given a parent directory, then it doesn't also need to be given + child directories, because it will naturally spider the contents of the parent directory. And + there are cases where Borg coming across the same file twice will result in duplicate reads and + even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to + Borg. + ''' + deduplicated = set() + + for directory in sorted(directories): + # If the directory is "/", that contains all child directories, so we can early out. + if directory == os.path.sep: + return (os.path.sep,) + + # If no other directories are parents of current directory (even n levels up), then the + # current directory isn't a duplicate. + if not any( + pathlib.PurePath(other_directory) in pathlib.PurePath(directory).parents + for other_directory in directories + ): + deduplicated.add(directory) + + return tuple(sorted(deduplicated)) + + def _write_pattern_file(patterns=None): ''' Given a sequence of patterns, write them to a named temporary file and return it. Return None @@ -148,9 +178,11 @@ def create_archive( If a sequence of stream processes is given (instances of subprocess.Popen), then execute the create command while also triggering the given processes to produce output. ''' - sources = _expand_directories( - location_config['source_directories'] - + borgmatic_source_directories(location_config.get('borgmatic_source_directory')) + sources = deduplicate_directories( + _expand_directories( + location_config['source_directories'] + + borgmatic_source_directories(location_config.get('borgmatic_source_directory')) + ) ) pattern_file = _write_pattern_file(location_config.get('patterns')) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index c36e213..6335a30 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -63,7 +63,7 @@ sudo pip3 install --upgrade borgmatic The main downside of a global install is that borgmatic is less cleanly separated from the rest of your Python software, and there's the theoretical -possibility for libary conflicts. But if you're okay with that, for instance +possibility of libary conflicts. But if you're okay with that, for instance on a relatively dedicated system, then a global install can work out just fine. diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 6b8d57d..0685953 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -60,6 +60,26 @@ def test_expand_home_directories_considers_none_as_no_directories(): assert paths == () +@pytest.mark.parametrize( + 'directories,expected_directories', + ( + (('/', '/root'), ('/',)), + (('/', '/root/'), ('/',)), + (('/root', '/'), ('/',)), + (('/root', '/root/foo'), ('/root',)), + (('/root/', '/root/foo'), ('/root/',)), + (('/root', '/root/foo/'), ('/root',)), + (('/root/foo', '/root'), ('/root',)), + (('/root', '/etc', '/root/foo/bar'), ('/etc', '/root')), + (('/root', '/root/foo', '/root/foo/bar'), ('/root',)), + (('/dup', '/dup'), ('/dup',)), + (('/foo', '/bar'), ('/bar', '/foo')), + ), +) +def test_deduplicate_directories_removes_child_paths(directories, expected_directories): + assert module.deduplicate_directories(directories) == expected_directories + + def test_write_pattern_file_does_not_raise(): temporary_file = flexmock(name='filename', write=lambda mode: None, flush=lambda: None) flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file) @@ -214,7 +234,8 @@ ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar') def test_create_archive_calls_borg_with_parameters(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -241,7 +262,8 @@ def test_create_archive_calls_borg_with_parameters(): def test_create_archive_with_patterns_calls_borg_with_patterns(): pattern_flags = ('--patterns-from', 'patterns') flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return( flexmock(name='/tmp/patterns') @@ -270,7 +292,8 @@ def test_create_archive_with_patterns_calls_borg_with_patterns(): def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): exclude_flags = ('--exclude-from', 'excludes') flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',)) flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return( flexmock(name='/tmp/excludes') @@ -298,7 +321,8 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): def test_create_archive_with_log_info_calls_borg_with_info_parameter(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -326,7 +350,8 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -355,7 +380,8 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -382,7 +408,8 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -410,7 +437,8 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -439,7 +467,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete # --dry-run and --stats are mutually exclusive, see: # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -468,7 +497,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -494,7 +524,8 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -520,7 +551,8 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param def test_create_archive_with_compression_calls_borg_with_compression_parameters(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -546,7 +578,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -572,7 +605,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -599,7 +633,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -626,7 +661,8 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet def test_create_archive_with_read_special_calls_borg_with_read_special_parameter(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -654,7 +690,8 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags')) def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -682,7 +719,8 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags')) def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -709,7 +747,8 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -736,7 +775,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( def test_create_archive_with_local_path_calls_borg_via_local_path(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -763,7 +803,8 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -790,7 +831,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( def test_create_archive_with_umask_calls_borg_with_umask_parameters(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -816,7 +858,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -842,7 +885,8 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_output_log_level(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -869,7 +913,8 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_and_info_output_log_level(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -897,7 +942,8 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_output_log_level(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -924,7 +970,8 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_and_info_output_log_level(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -952,7 +999,8 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -980,7 +1028,8 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para def test_create_archive_with_progress_calls_borg_with_progress_parameter(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -1008,7 +1057,8 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_parameter(): processes = flexmock() flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -1037,7 +1087,8 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr def test_create_archive_with_json_calls_borg_with_json_parameter(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -1066,7 +1117,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -1096,7 +1148,8 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() def test_create_archive_with_source_directories_glob_expands(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -1123,7 +1176,8 @@ def test_create_archive_with_source_directories_glob_expands(): def test_create_archive_with_non_matching_source_directories_glob_passes_through(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo*',)) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo*',)) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -1150,7 +1204,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through def test_create_archive_with_glob_calls_borg_with_expanded_directories(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -1176,7 +1231,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -1202,7 +1258,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -1228,7 +1285,8 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(()) @@ -1255,7 +1313,8 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): def test_create_archive_with_stream_processes_calls_borg_with_processes(): processes = flexmock() flexmock(module).should_receive('borgmatic_source_directories').and_return([]) - flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('_expand_directories').and_return(()) flexmock(module).should_receive('_expand_home_directories').and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) flexmock(module).should_receive('_make_pattern_flags').and_return(())