psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix hang when streaming a database dump to Borg with implicit duplicate source directories by deduplicating them first (#316).

Browse source
This commit is contained in:
Dan Helfman 2020-05-20 13:33:53 -07:00
parent 244dc35bae
commit 96df52ec50
4 changed files with 136 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
NEWS
View file

@ -1,6 +1,8 @@
1.5.5.dev0
* #314: Fix regression in support for PostgreSQL's "directory" dump format. Unlike other dump
formats, the "directory" dump format does not stream directly to/from Borg.
* #316: Fix hang when streaming a database dump to Borg with implicit duplicate source directories
by deduplicating them first.
* Improve documentation around the installation process. Specifically, making borgmatic commands
runnable via the system PATH and offering a global install option.

34
borgmatic/borg/create.py
View file

@ -2,6 +2,7 @@ import glob
import itertools
import logging
import os
import pathlib
import tempfile
from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_with_processes
@ -43,6 +44,35 @@ def _expand_home_directories(directories):
return tuple(os.path.expanduser(directory) for directory in directories)
def deduplicate_directories(directories):
'''
Given a sequence of directories, return them as a sorted tuple with all duplicate child
directories removed. For instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
The idea is that if Borg is given a parent directory, then it doesn't also need to be given
child directories, because it will naturally spider the contents of the parent directory. And
there are cases where Borg coming across the same file twice will result in duplicate reads and
even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
Borg.
'''
deduplicated = set()
for directory in sorted(directories):
# If the directory is "/", that contains all child directories, so we can early out.
if directory == os.path.sep:
return (os.path.sep,)
# If no other directories are parents of current directory (even n levels up), then the
# current directory isn't a duplicate.
if not any(
pathlib.PurePath(other_directory) in pathlib.PurePath(directory).parents
for other_directory in directories
):
deduplicated.add(directory)
return tuple(sorted(deduplicated))
def _write_pattern_file(patterns=None):
'''
Given a sequence of patterns, write them to a named temporary file and return it. Return None
@ -148,10 +178,12 @@ def create_archive(
If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
create command while also triggering the given processes to produce output.
'''
sources = _expand_directories(
sources = deduplicate_directories(
_expand_directories(
location_config['source_directories']
+ borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
)
)
pattern_file = _write_pattern_file(location_config.get('patterns'))
exclude_file = _write_pattern_file(

2
docs/how-to/set-up-backups.md
View file

@ -63,7 +63,7 @@ sudo pip3 install --upgrade borgmatic
The main downside of a global install is that borgmatic is less cleanly
separated from the rest of your Python software, and there's the theoretical
possibility for libary conflicts. But if you're okay with that, for instance
possibility of libary conflicts. But if you're okay with that, for instance
on a relatively dedicated system, then a global install can work out just
fine.

137
tests/unit/borg/test_create.py
View file

@ -60,6 +60,26 @@ def test_expand_home_directories_considers_none_as_no_directories():
assert paths == ()
@pytest.mark.parametrize(
'directories,expected_directories',
(
(('/', '/root'), ('/',)),
(('/', '/root/'), ('/',)),
(('/root', '/'), ('/',)),
(('/root', '/root/foo'), ('/root',)),
(('/root/', '/root/foo'), ('/root/',)),
(('/root', '/root/foo/'), ('/root',)),
(('/root/foo', '/root'), ('/root',)),
(('/root', '/etc', '/root/foo/bar'), ('/etc', '/root')),
(('/root', '/root/foo', '/root/foo/bar'), ('/root',)),
(('/dup', '/dup'), ('/dup',)),
(('/foo', '/bar'), ('/bar', '/foo')),
),
)
def test_deduplicate_directories_removes_child_paths(directories, expected_directories):
assert module.deduplicate_directories(directories) == expected_directories
def test_write_pattern_file_does_not_raise():
temporary_file = flexmock(name='filename', write=lambda mode: None, flush=lambda: None)
flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
@ -214,7 +234,8 @@ ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
def test_create_archive_calls_borg_with_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -241,7 +262,8 @@ def test_create_archive_calls_borg_with_parameters():
def test_create_archive_with_patterns_calls_borg_with_patterns():
pattern_flags = ('--patterns-from', 'patterns')
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(
flexmock(name='/tmp/patterns')
@ -270,7 +292,8 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
exclude_flags = ('--exclude-from', 'excludes')
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
flexmock(name='/tmp/excludes')
@ -298,7 +321,8 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
def test_create_archive_with_log_info_calls_borg_with_info_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -326,7 +350,8 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -355,7 +380,8 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -382,7 +408,8 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -410,7 +437,8 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -439,7 +467,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
# --dry-run and --stats are mutually exclusive, see:
# https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -468,7 +497,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -494,7 +524,8 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -520,7 +551,8 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
def test_create_archive_with_compression_calls_borg_with_compression_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -546,7 +578,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -572,7 +605,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -599,7 +633,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -626,7 +661,8 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -654,7 +690,8 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -682,7 +719,8 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -709,7 +747,8 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -736,7 +775,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
def test_create_archive_with_local_path_calls_borg_via_local_path():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -763,7 +803,8 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -790,7 +831,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
def test_create_archive_with_umask_calls_borg_with_umask_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -816,7 +858,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -842,7 +885,8 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_output_log_level():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -869,7 +913,8 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_and_info_output_log_level():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -897,7 +942,8 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_output_log_level():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -924,7 +970,8 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_and_info_output_log_level():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -952,7 +999,8 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -980,7 +1028,8 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
def test_create_archive_with_progress_calls_borg_with_progress_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -1008,7 +1057,8 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_parameter():
processes = flexmock()
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -1037,7 +1087,8 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
def test_create_archive_with_json_calls_borg_with_json_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -1066,7 +1117,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -1096,7 +1148,8 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
def test_create_archive_with_source_directories_glob_expands():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -1123,7 +1176,8 @@ def test_create_archive_with_source_directories_glob_expands():
def test_create_archive_with_non_matching_source_directories_glob_passes_through():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo*',))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo*',))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -1150,7 +1204,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
def test_create_archive_with_glob_calls_borg_with_expanded_directories():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -1176,7 +1231,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -1202,7 +1258,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -1228,7 +1285,8 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())
@ -1255,7 +1313,8 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
def test_create_archive_with_stream_processes_calls_borg_with_processes():
processes = flexmock()
flexmock(module).should_receive('borgmatic_source_directories').and_return([])
flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
flexmock(module).should_receive('_expand_directories').and_return(())
flexmock(module).should_receive('_expand_home_directories').and_return(())
flexmock(module).should_receive('_write_pattern_file').and_return(None)
flexmock(module).should_receive('_make_pattern_flags').and_return(())