From 10cac46f4c56f47b61e1b938a4f8443f39c369d6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 5 Aug 2017 23:32:39 -0700 Subject: [PATCH] #12, #35: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options. --- NEWS | 4 + README.md | 2 +- borgmatic/borg/create.py | 32 +++++++- borgmatic/config/schema.yaml | 18 +++++ borgmatic/tests/unit/borg/test_create.py | 96 +++++++++++++++++++++--- setup.py | 2 +- 6 files changed, 136 insertions(+), 18 deletions(-) diff --git a/NEWS b/NEWS index 6222fea..66447b2 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.1.6 + + * #12, #35: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options. + 1.1.5 * #34: New "extract" consistency check that performs a dry-run extraction of the most recent diff --git a/README.md b/README.md index 057388b..6840d06 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ However, see below about special cases. borgmatic changed its configuration file format in version 1.1.0 from INI-style to YAML. This better supports validation, and has a more natural way to express lists of values. To upgrade your existing configuration, first -upgrade to the new version of borgmatic: +upgrade to the new version of borgmatic. As of version 1.1.0, borgmatic no longer supports Python 2. If you were already running borgmatic with Python 3, then you can simply upgrade borgmatic diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 21af701..2696e75 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -31,12 +31,33 @@ def _write_exclude_file(exclude_patterns=None): return exclude_file +def _make_exclude_flags(location_config, exclude_patterns_filename=None): + ''' + Given a location config dict with various exclude options, and a filename containing any exclude + patterns, return the corresponding Borg flags as a tuple. + ''' + exclude_filenames = tuple(location_config.get('exclude_from', ())) + ( + (exclude_patterns_filename,) if exclude_patterns_filename else () + ) + exclude_from_flags = tuple( + itertools.chain.from_iterable( + ('--exclude-from', exclude_filename) + for exclude_filename in exclude_filenames + ) + ) + caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else () + if_present = location_config.get('exclude_if_present') + if_present_flags = ('--exclude-if-present', if_present) if if_present else () + + return exclude_from_flags + caches_flag + if_present_flags + + def create_archive( verbosity, repository, location_config, storage_config, ): ''' - Given a vebosity flag, a storage config dict, a list of source directories, a local or remote - repository path, a list of exclude patterns, create a Borg archive. + Given a vebosity flag, a local or remote repository path, a location config dict, and a storage + config dict, create a Borg archive. ''' sources = tuple( itertools.chain.from_iterable( @@ -45,8 +66,11 @@ def create_archive( ) ) - exclude_file = _write_exclude_file(location_config.get('exclude_patterns')) - exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else () + exclude_patterns_file = _write_exclude_file(location_config.get('exclude_patterns')) + exclude_flags = _make_exclude_flags( + location_config, + exclude_patterns_file.name if exclude_patterns_file else None, + ) compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () umask = storage_config.get('umask', None) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 567ec73..f7e4c4f 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -45,6 +45,24 @@ map: - '*.pyc' - /home/*/.cache - /etc/ssl + exclude_from: + seq: + - type: scalar + desc: | + Read exclude patterns from one or more separate named files, one pattern per + line. + example: + - /etc/borgmatic/excludes + exclude_caches: + type: bool + desc: | + Exclude directories that contain a CACHEDIR.TAG file. See + http://www.brynosaurus.com/cachedir/spec.html for details. + example: true + exclude_if_present: + type: scalar + desc: Exclude directories that contain a file with the given filename. + example: .nobackup storage: desc: | Repository storage options. See diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py index 3b4d268..6d2aa45 100644 --- a/borgmatic/tests/unit/borg/test_create.py +++ b/borgmatic/tests/unit/borg/test_create.py @@ -58,11 +58,72 @@ def insert_datetime_mock(): ).mock +def test_make_exclude_flags_includes_exclude_patterns_filename_when_given(): + exclude_flags = module._make_exclude_flags( + location_config={'exclude_patterns': ['*.pyc', '/var']}, + exclude_patterns_filename='/tmp/excludes', + ) + + assert exclude_flags == ('--exclude-from', '/tmp/excludes') + + +def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config(): + flexmock(module).should_receive('_write_exclude_file').and_return(None) + + exclude_flags = module._make_exclude_flags( + location_config={'exclude_from': ['excludes', 'other']}, + ) + + assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', 'other') + + +def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config(): + flexmock(module).should_receive('_write_exclude_file').and_return(None) + + exclude_flags = module._make_exclude_flags( + location_config={'exclude_from': ['excludes']}, + exclude_patterns_filename='/tmp/excludes', + ) + + assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes') + + +def test_make_exclude_flags_includes_exclude_caches_when_true_in_config(): + exclude_flags = module._make_exclude_flags( + location_config={'exclude_caches': True}, + ) + + assert exclude_flags == ('--exclude-caches',) + + +def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config(): + exclude_flags = module._make_exclude_flags( + location_config={'exclude_caches': False}, + ) + + assert exclude_flags == () + + +def test_make_exclude_flags_includes_exclude_if_present_when_in_config(): + exclude_flags = module._make_exclude_flags( + location_config={'exclude_if_present': 'exclude_me'}, + ) + + assert exclude_flags == ('--exclude-if-present', 'exclude_me') + + +def test_make_exclude_flags_is_empty_when_config_has_no_excludes(): + exclude_flags = module._make_exclude_flags(location_config={}) + + assert exclude_flags == () + + CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar') def test_create_archive_should_call_borg_with_parameters(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND) insert_platform_mock() insert_datetime_mock() @@ -80,8 +141,10 @@ def test_create_archive_should_call_borg_with_parameters(): def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): - flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes')) - insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes')) + exclude_flags = ('--exclude-from', 'excludes') + flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes')) + flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags) + insert_subprocess_mock(CREATE_COMMAND + exclude_flags) insert_platform_mock() insert_datetime_mock() @@ -98,7 +161,8 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',)) insert_platform_mock() insert_datetime_mock() @@ -116,7 +180,8 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats')) insert_platform_mock() insert_datetime_mock() @@ -134,7 +199,8 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete def test_create_archive_with_compression_should_call_borg_with_compression_parameters(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) insert_platform_mock() insert_datetime_mock() @@ -152,7 +218,8 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) insert_platform_mock() insert_datetime_mock() @@ -171,7 +238,8 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1')) insert_platform_mock() insert_datetime_mock() @@ -190,7 +258,8 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) insert_platform_mock() insert_datetime_mock() @@ -208,7 +277,8 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): def test_create_archive_with_source_directories_glob_expands(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() @@ -227,7 +297,8 @@ def test_create_archive_with_source_directories_glob_expands(): def test_create_archive_with_non_matching_source_directories_glob_passes_through(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*')) insert_platform_mock() insert_datetime_mock() @@ -246,7 +317,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() diff --git a/setup.py b/setup.py index 2ff9690..a842ab5 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.5' +VERSION = '1.1.6' setup(