Support for using tilde in exclude_patterns to reference home directory (#58).

NEWS

@@ -1,7 +1,8 @@
 1.1.15.dev0
  * Support for Borg BORG_PASSCOMMAND environment variable to read a password from an external file.
- * #55: Fix for missing tags/releases from Gitea and GitHub project hosting.
  * Fix for Borg create error when using borgmatic's --dry-run and --verbosity options together.
+ * #55: Fix for missing tags/releases from Gitea and GitHub project hosting.
+ * #58: Support for using tilde in exclude_patterns to reference home directory.
 
 1.1.14
  * #49: Fix for typo in --patterns-from option.

borgmatic/borg/create.py

@@ -35,6 +35,22 @@ def _expand_directory(directory):
     return glob.glob(expanded_directory) or [expanded_directory]
 
 
+def _expand_directories(directories):
+    '''
+    Given a sequence of directory paths, expand tildes and globs in each one. Return all the
+    resulting directories as a single flattened tuple.
+    '''
+    if directories is None:
+        return ()
+
+    return tuple(
+        itertools.chain.from_iterable(
+            _expand_directory(directory)
+            for directory in directories
+        )
+    )
+
+
 def _write_pattern_file(patterns=None):
     '''
     Given a sequence of patterns, write them to a named temporary file and return it. Return None
@@ -95,19 +111,14 @@ def create_archive(
     Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
     storage config dict, create a Borg archive.
     '''
-    sources = tuple(
+    sources = _expand_directories(location_config['source_directories'])
-        itertools.chain.from_iterable(
-            _expand_directory(directory)
-            for directory in location_config['source_directories']
-        )
-    )
 
     pattern_file = _write_pattern_file(location_config.get('patterns'))
     pattern_flags = _make_pattern_flags(
         location_config,
         pattern_file.name if pattern_file else None,
     )
-    exclude_file = _write_pattern_file(location_config.get('exclude_patterns'))
+    exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns')))
     exclude_flags = _make_exclude_flags(
         location_config,
         exclude_file.name if exclude_file else None,

borgmatic/config/schema.yaml

@@ -52,9 +52,9 @@ map:
                     - type: scalar
                 desc: |
                     Any paths matching these patterns are included/excluded from backups. Globs are
-                    expanded. Note that Borg considers this option experimental. See the output of
+                    expanded. (Tildes are not.) Note that Borg considers this option experimental.
-                    "borg help patterns" for more details. Quoting any value if it contains leading
+                    See the output of "borg help patterns" for more details. Quote any value if it
-                    punctuation, so it parses correctly.
+                    contains leading punctuation, so it parses correctly.
                 example:
                     - 'R /'
                     - '- /home/*/.cache'
@@ -73,11 +73,11 @@ map:
                 seq:
                     - type: scalar
                 desc: |
-                    Any paths matching these patterns are excluded from backups. Globs are expanded.
+                    Any paths matching these patterns are excluded from backups. Globs and tildes
-                    See the output of "borg help patterns" for more details.
+                    are expanded. See the output of "borg help patterns" for more details.
                 example:
                     - '*.pyc'
-                    - /home/*/.cache
+                    - ~/*/.cache
                     - /etc/ssl
             exclude_from:
                 seq:

borgmatic/tests/unit/borg/test_create.py

@@ -70,6 +70,21 @@ def test_expand_directory_with_glob_expands():
     assert paths == ['foo', 'food']
 
 
+def test_expand_directories_flattens_expanded_directories():
+    flexmock(module).should_receive('_expand_directory').with_args('~/foo').and_return(['/root/foo'])
+    flexmock(module).should_receive('_expand_directory').with_args('bar*').and_return(['bar', 'barf'])
+
+    paths = module._expand_directories(('~/foo', 'bar*'))
+
+    assert paths == ('/root/foo', 'bar', 'barf')
+
+
+def test_expand_directories_considers_none_as_no_directories():
+    paths = module._expand_directories(None)
+
+    assert paths == ()
+
+
 def test_write_pattern_file_does_not_raise():
     temporary_file = flexmock(
         name='filename',
@@ -194,7 +209,7 @@ CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'fo
 
 
 def test_create_archive_calls_borg_with_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -215,7 +230,7 @@ def test_create_archive_calls_borg_with_parameters():
 
 def test_create_archive_with_patterns_calls_borg_with_patterns():
     pattern_flags = ('--patterns-from', 'patterns')
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(flexmock(name='/tmp/patterns')).and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -236,7 +251,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
 
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(('exclude',))
     flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(flexmock(name='/tmp/excludes'))
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
@@ -256,7 +271,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 
 
 def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -277,7 +292,7 @@ def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter():
 
 
 def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -297,7 +312,7 @@ def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter():
 
 
 def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -318,7 +333,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
 
 
 def test_create_archive_with_dry_run_and_verbosity_some_calls_borg_without_stats_parameter():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -339,7 +354,7 @@ def test_create_archive_with_dry_run_and_verbosity_some_calls_borg_without_stats
 
 
 def test_create_archive_with_dry_run_and_verbosity_lots_calls_borg_without_stats_parameter():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -360,7 +375,7 @@ def test_create_archive_with_dry_run_and_verbosity_lots_calls_borg_without_stats
 
 
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -380,7 +395,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 
 
 def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -400,7 +415,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
 
 
 def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -421,7 +436,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
 
 
 def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -442,7 +457,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 
 
 def test_create_archive_with_local_path_calls_borg_via_local_path():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -463,7 +478,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
 
 
 def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -484,7 +499,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 
 
 def test_create_archive_with_umask_calls_borg_with_umask_parameters():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -504,7 +519,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
 
 
 def test_create_archive_with_source_directories_glob_expands():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -525,7 +540,7 @@ def test_create_archive_with_source_directories_glob_expands():
 
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo*'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo*',)).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -546,7 +561,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 
 def test_create_archive_with_glob_calls_borg_with_expanded_directories():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -566,7 +581,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 
 
 def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
@@ -588,7 +603,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 
 
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
-    flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar'])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return([])
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())