Fix hang when streaming a database dump to Borg with implicit duplicate source directories by deduplicating them first (#316).

NEWS

@@ -1,6 +1,8 @@
 1.5.5.dev0
  * #314: Fix regression in support for PostgreSQL's "directory" dump format. Unlike other dump
    formats, the "directory" dump format does not stream directly to/from Borg.
+ * #316: Fix hang when streaming a database dump to Borg with implicit duplicate source directories
+   by deduplicating them first.
  * Improve documentation around the installation process. Specifically, making borgmatic commands
    runnable via the system PATH and offering a global install option.
 

borgmatic/borg/create.py

@@ -2,6 +2,7 @@ import glob
 import itertools
 import logging
 import os
+import pathlib
 import tempfile
 
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_with_processes
@@ -43,6 +44,35 @@ def _expand_home_directories(directories):
     return tuple(os.path.expanduser(directory) for directory in directories)
 
 
+def deduplicate_directories(directories):
+    '''
+    Given a sequence of directories, return them as a sorted tuple with all duplicate child
+    directories removed. For instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
+
+    The idea is that if Borg is given a parent directory, then it doesn't also need to be given
+    child directories, because it will naturally spider the contents of the parent directory. And
+    there are cases where Borg coming across the same file twice will result in duplicate reads and
+    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
+    Borg.
+    '''
+    deduplicated = set()
+
+    for directory in sorted(directories):
+        # If the directory is "/", that contains all child directories, so we can early out.
+        if directory == os.path.sep:
+            return (os.path.sep,)
+
+        # If no other directories are parents of current directory (even n levels up), then the
+        # current directory isn't a duplicate.
+        if not any(
+            pathlib.PurePath(other_directory) in pathlib.PurePath(directory).parents
+            for other_directory in directories
+        ):
+            deduplicated.add(directory)
+
+    return tuple(sorted(deduplicated))
+
+
 def _write_pattern_file(patterns=None):
     '''
     Given a sequence of patterns, write them to a named temporary file and return it. Return None
@@ -148,10 +178,12 @@ def create_archive(
     If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
     create command while also triggering the given processes to produce output.
     '''
-    sources = _expand_directories(
+    sources = deduplicate_directories(
+        _expand_directories(
             location_config['source_directories']
             + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
         )
+    )
 
     pattern_file = _write_pattern_file(location_config.get('patterns'))
     exclude_file = _write_pattern_file(

docs/how-to/set-up-backups.md

@@ -63,7 +63,7 @@ sudo pip3 install --upgrade borgmatic
 
 The main downside of a global install is that borgmatic is less cleanly
 separated from the rest of your Python software, and there's the theoretical
-possibility for libary conflicts. But if you're okay with that, for instance
+possibility of libary conflicts. But if you're okay with that, for instance
 on a relatively dedicated system, then a global install can work out just
 fine.
 

tests/unit/borg/test_create.py

@@ -60,6 +60,26 @@ def test_expand_home_directories_considers_none_as_no_directories():
     assert paths == ()
 
 
+@pytest.mark.parametrize(
+    'directories,expected_directories',
+    (
+        (('/', '/root'), ('/',)),
+        (('/', '/root/'), ('/',)),
+        (('/root', '/'), ('/',)),
+        (('/root', '/root/foo'), ('/root',)),
+        (('/root/', '/root/foo'), ('/root/',)),
+        (('/root', '/root/foo/'), ('/root',)),
+        (('/root/foo', '/root'), ('/root',)),
+        (('/root', '/etc', '/root/foo/bar'), ('/etc', '/root')),
+        (('/root', '/root/foo', '/root/foo/bar'), ('/root',)),
+        (('/dup', '/dup'), ('/dup',)),
+        (('/foo', '/bar'), ('/bar', '/foo')),
+    ),
+)
+def test_deduplicate_directories_removes_child_paths(directories, expected_directories):
+    assert module.deduplicate_directories(directories) == expected_directories
+
+
 def test_write_pattern_file_does_not_raise():
     temporary_file = flexmock(name='filename', write=lambda mode: None, flush=lambda: None)
     flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
@@ -214,7 +234,8 @@ ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
 
 def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -241,7 +262,8 @@ def test_create_archive_calls_borg_with_parameters():
 def test_create_archive_with_patterns_calls_borg_with_patterns():
     pattern_flags = ('--patterns-from', 'patterns')
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(
         flexmock(name='/tmp/patterns')
@@ -270,7 +292,8 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
     flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
         flexmock(name='/tmp/excludes')
@@ -298,7 +321,8 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 
 def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -326,7 +350,8 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
 
 def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -355,7 +380,8 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
 
 def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -382,7 +408,8 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -410,7 +437,8 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
 
 def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -439,7 +467,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
     # --dry-run and --stats are mutually exclusive, see:
     # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -468,7 +497,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
 
 def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -494,7 +524,8 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
 
 def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -520,7 +551,8 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
 
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -546,7 +578,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 
 def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -572,7 +605,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
 
 def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -599,7 +633,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
 
 def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -626,7 +661,8 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
 
 def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -654,7 +690,8 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
 @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
 def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -682,7 +719,8 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
 @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
 def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -709,7 +747,8 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
 
 def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -736,7 +775,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 
 def test_create_archive_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -763,7 +803,8 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
 
 def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -790,7 +831,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 
 def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -816,7 +858,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
 
 def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -842,7 +885,8 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
 
 def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -869,7 +913,8 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
 
 def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_and_info_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -897,7 +942,8 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
 
 def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -924,7 +970,8 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
 
 def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_and_info_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -952,7 +999,8 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
 
 def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -980,7 +1028,8 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
 
 def test_create_archive_with_progress_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1008,7 +1057,8 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
 def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_parameter():
     processes = flexmock()
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1037,7 +1087,8 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
 
 def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1066,7 +1117,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
 
 def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1096,7 +1148,8 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
 
 def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1123,7 +1176,8 @@ def test_create_archive_with_source_directories_glob_expands():
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo*',))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo*',))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1150,7 +1204,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 def test_create_archive_with_glob_calls_borg_with_expanded_directories():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1176,7 +1231,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 
 def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1202,7 +1258,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1228,7 +1285,8 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
 
 def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1255,7 +1313,8 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
 def test_create_archive_with_stream_processes_calls_borg_with_processes():
     processes = flexmock()
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())