psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixing up borg module to deal with new parsed config file structures.

Browse source
This commit is contained in:
Dan Helfman 2017-07-22 20:11:49 -07:00
parent fb172f018a
commit 8b2b41eefc
3 changed files with 95 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
borgmatic/borg.py
View file

@ -1,10 +1,11 @@
from datetime import datetime
import glob
import itertools
import os
import re
import platform
import re
import subprocess
from glob import glob
from itertools import chain
import tempfile
from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
@ -22,18 +23,38 @@ def initialize(storage_config, command=COMMAND):
os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase
def _write_exclude_file(exclude_patterns=None):
'''
Given a sequence of exclude patterns, write them to a named temporary file and return it. Return
None if no patterns are provided.
'''
if not exclude_patterns:
return None
exclude_file = tempfile.NamedTemporaryFile('w')
exclude_file.write('\n'.join(exclude_patterns))
exclude_file.flush()
return exclude_file
def create_archive(
excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND,
one_file_system=None, remote_path=None,
verbosity, storage_config, source_directories, repository, exclude_patterns=None,
command=COMMAND, one_file_system=None, remote_path=None,
):
'''
Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
list of source directories, a local or remote repository path, and a command to run, create an
attic archive.
Given a vebosity flag, a storage config dict, a list of source directories, a local or remote
repository path, a list of exclude patterns, and a command to run, create an attic archive.
'''
sources = re.split('\s+', source_directories)
sources = tuple(chain.from_iterable(glob(x) or [x] for x in sources))
exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
sources = tuple(
itertools.chain.from_iterable(
glob.glob(directory) or [directory]
for directory in source_directories
)
)
exclude_file = _write_exclude_file(exclude_patterns)
exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else ()
compression = storage_config.get('compression', None)
compression_flags = ('--compression', compression) if compression else ()
umask = storage_config.get('umask', None)
@ -109,12 +130,11 @@ DEFAULT_CHECKS = ('repository', 'archives')
def _parse_checks(consistency_config):
'''
Given a consistency config with a space-separated "checks" option, transform it to a tuple of
named checks to run.
Given a consistency config with a "checks" list, transform it to a tuple of named checks to run.
For example, given a retention config of:
{'checks': 'repository archives'}
{'checks': ['repository', 'archives']}
This will be returned as:
@ -123,14 +143,11 @@ def _parse_checks(consistency_config):
If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
"disabled", return an empty tuple, meaning that no checks should be run.
'''
checks = consistency_config.get('checks', '').strip()
if not checks:
return DEFAULT_CHECKS
checks = consistency_config.get('checks', [])
if checks == ['disabled']:
return ()
return tuple(
check for check in consistency_config['checks'].split(' ')
if check.lower() not in ('disabled', '')
)
return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS
def _make_check_flags(checks, check_last=None):

5
borgmatic/commands/borgmatic.py
View file

@ -48,10 +48,7 @@ def main(): # pragma: no cover
remote_path = config.location['remote_path']
borg.initialize(config.storage)
# TODO: Use the new exclude_patterns.
borg.create_archive(
args.excludes_filename, args.verbosity, config.storage, **config.location
)
borg.create_archive(args.verbosity, config.storage, **config.location)
borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path)
borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path)
except (ValueError, OSError, CalledProcessError) as error:

103
borgmatic/tests/unit/test_borg.py
View file

@ -30,6 +30,20 @@ def test_initialize_without_passphrase_should_not_set_environment():
finally:
os.environ = orig_environ
def test_write_exclude_file_does_not_raise():
temporary_file = flexmock(
name='filename',
write=lambda mode: None,
flush=lambda: None,
)
flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
module._write_exclude_file(['exclude'])
def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise():
module._write_exclude_file([])
def insert_subprocess_mock(check_call_command, **kwargs):
subprocess = flexmock(STDOUT=STDOUT)
@ -53,110 +67,100 @@ def insert_datetime_mock():
).mock
CREATE_COMMAND_WITHOUT_EXCLUDES = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes')
CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar')
def test_create_archive_should_call_borg_with_parameters():
flexmock(module).should_receive('_write_exclude_file')
insert_subprocess_mock(CREATE_COMMAND)
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
exclude_patterns=None,
verbosity=None,
storage_config={},
source_directories='foo bar',
source_directories=['foo', 'bar'],
repository='repo',
command='borg',
)
def test_create_archive_with_two_spaces_in_source_directories():
insert_subprocess_mock(CREATE_COMMAND)
def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes():
flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes'))
insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes'))
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
exclude_patterns=['exclude'],
verbosity=None,
storage_config={},
source_directories='foo bar',
repository='repo',
command='borg',
)
def test_create_archive_with_none_excludes_filename_should_call_borg_without_excludes():
insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES)
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename=None,
verbosity=None,
storage_config={},
source_directories='foo bar',
source_directories=['foo', 'bar'],
repository='repo',
command='borg',
)
def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter():
flexmock(module).should_receive('_write_exclude_file')
insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',))
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
exclude_patterns=None,
verbosity=VERBOSITY_SOME,
storage_config={},
source_directories='foo bar',
source_directories=['foo', 'bar'],
repository='repo',
command='borg',
)
def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter():
flexmock(module).should_receive('_write_exclude_file')
insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats'))
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
exclude_patterns=None,
verbosity=VERBOSITY_LOTS,
storage_config={},
source_directories='foo bar',
source_directories=['foo', 'bar'],
repository='repo',
command='borg',
)
def test_create_archive_with_compression_should_call_borg_with_compression_parameters():
flexmock(module).should_receive('_write_exclude_file')
insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
exclude_patterns=None,
verbosity=None,
storage_config={'compression': 'rle'},
source_directories='foo bar',
source_directories=['foo', 'bar'],
repository='repo',
command='borg',
)
def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters():
flexmock(module).should_receive('_write_exclude_file')
insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
exclude_patterns=None,
verbosity=None,
storage_config={},
source_directories='foo bar',
source_directories=['foo', 'bar'],
repository='repo',
command='borg',
one_file_system=True,
@ -164,15 +168,16 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst
def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters():
flexmock(module).should_receive('_write_exclude_file')
insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
exclude_patterns=None,
verbosity=None,
storage_config={},
source_directories='foo bar',
source_directories=['foo', 'bar'],
repository='repo',
command='borg',
remote_path='borg1',
@ -180,63 +185,67 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param
def test_create_archive_with_umask_should_call_borg_with_umask_parameters():
flexmock(module).should_receive('_write_exclude_file')
insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
exclude_patterns=None,
verbosity=None,
storage_config={'umask': 740},
source_directories='foo bar',
source_directories=['foo', 'bar'],
repository='repo',
command='borg',
)
def test_create_archive_with_source_directories_glob_expands():
flexmock(module).should_receive('_write_exclude_file')
insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
insert_platform_mock()
insert_datetime_mock()
flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
module.create_archive(
excludes_filename=None,
exclude_patterns=None,
verbosity=None,
storage_config={},
source_directories='foo*',
source_directories=['foo*'],
repository='repo',
command='borg',
)
def test_create_archive_with_non_matching_source_directories_glob_passes_through():
flexmock(module).should_receive('_write_exclude_file')
insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*'))
insert_platform_mock()
insert_datetime_mock()
flexmock(module).should_receive('glob').with_args('foo*').and_return([])
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
module.create_archive(
excludes_filename=None,
exclude_patterns=None,
verbosity=None,
storage_config={},
source_directories='foo*',
source_directories=['foo*'],
repository='repo',
command='borg',
)
def test_create_archive_with_glob_should_call_borg_with_expanded_directories():
flexmock(module).should_receive('_write_exclude_file')
insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food'))
insert_platform_mock()
insert_datetime_mock()
flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
module.create_archive(
excludes_filename=None,
exclude_patterns=None,
verbosity=None,
storage_config={},
source_directories='foo*',
source_directories=['foo*'],
repository='repo',
command='borg',
)
@ -329,7 +338,7 @@ def test_prune_archive_with_remote_path_should_call_borg_with_remote_path_parame
def test_parse_checks_returns_them_as_tuple():
checks = module._parse_checks({'checks': 'foo disabled bar'})
checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']})
assert checks == ('foo', 'bar')
@ -341,13 +350,13 @@ def test_parse_checks_with_missing_value_returns_defaults():
def test_parse_checks_with_blank_value_returns_defaults():
checks = module._parse_checks({'checks': ''})
checks = module._parse_checks({'checks': []})
assert checks == module.DEFAULT_CHECKS
def test_parse_checks_with_disabled_returns_no_checks():
checks = module._parse_checks({'checks': 'disabled'})
checks = module._parse_checks({'checks': ['disabled']})
assert checks == ()