#14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
This commit is contained in:
parent
57b3066987
commit
7ed5b33db5
9 changed files with 172 additions and 38 deletions
5
NEWS
5
NEWS
|
@ -1,3 +1,8 @@
|
|||
1.1.3.dev0
|
||||
|
||||
* #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
|
||||
* Fix for generate-borgmatic-config writing config with invalid one_file_system value.
|
||||
|
||||
1.1.2
|
||||
|
||||
* #32: Fix for passing check_last as integer to subprocess when calling Borg.
|
||||
|
|
23
README.md
23
README.md
|
@ -68,7 +68,9 @@ To install borgmatic, run the following command to download and install it:
|
|||
Make sure you're using Python 3, as borgmatic does not support Python 2. (You
|
||||
may have to use "pip3" or similar instead of "pip".)
|
||||
|
||||
Then, generate a sample configuration file:
|
||||
## Configuration
|
||||
|
||||
After you install borgmatic, generate a sample configuration file:
|
||||
|
||||
sudo generate-borgmatic-config
|
||||
|
||||
|
@ -78,6 +80,25 @@ representative. All fields are optional except where indicated, so feel free
|
|||
to remove anything you don't need.
|
||||
|
||||
|
||||
### Multiple configuration files
|
||||
|
||||
A more advanced usage is to create multiple separate configuration files and
|
||||
place each one in a /etc/borgmatic.d directory. For instance:
|
||||
|
||||
sudo mkdir /etc/borgmatic.d
|
||||
sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
|
||||
sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
|
||||
|
||||
With this approach, you can have entirely different backup policies for
|
||||
different applications on your system. For instance, you may want one backup
|
||||
configuration for your database data directory, and a different configuration
|
||||
for your user home directories.
|
||||
|
||||
When you set up multiple configuration files like this, borgmatic will run
|
||||
each one in turn from a single borgmatic invocation. This includes, by
|
||||
default, the traditional /etc/borgmatic/config.yaml as well.
|
||||
|
||||
|
||||
## Upgrading
|
||||
|
||||
In general, all you should need to do to upgrade borgmatic is run the
|
||||
|
|
|
@ -5,12 +5,12 @@ from subprocess import CalledProcessError
|
|||
import sys
|
||||
|
||||
from borgmatic import borg
|
||||
from borgmatic.config import convert, validate
|
||||
from borgmatic.config import collect, convert, validate
|
||||
|
||||
|
||||
LEGACY_CONFIG_FILENAME = '/etc/borgmatic/config'
|
||||
DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
|
||||
DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
|
||||
LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
|
||||
DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d']
|
||||
DEFAULT_EXCLUDES_PATH = '/etc/borgmatic/excludes'
|
||||
|
||||
|
||||
def parse_arguments(*arguments):
|
||||
|
@ -21,9 +21,10 @@ def parse_arguments(*arguments):
|
|||
parser = ArgumentParser()
|
||||
parser.add_argument(
|
||||
'-c', '--config',
|
||||
dest='config_filename',
|
||||
default=DEFAULT_CONFIG_FILENAME,
|
||||
help='Configuration filename',
|
||||
nargs='+',
|
||||
dest='config_paths',
|
||||
default=DEFAULT_CONFIG_PATHS,
|
||||
help='Configuration filenames or directories, defaults to: {}'.format(' '.join(DEFAULT_CONFIG_PATHS)),
|
||||
)
|
||||
parser.add_argument(
|
||||
'--excludes',
|
||||
|
@ -42,25 +43,31 @@ def parse_arguments(*arguments):
|
|||
def main(): # pragma: no cover
|
||||
try:
|
||||
args = parse_arguments(*sys.argv[1:])
|
||||
convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename)
|
||||
config = validate.parse_configuration(args.config_filename, validate.schema_filename())
|
||||
(location, storage, retention, consistency) = (
|
||||
config.get(section_name, {})
|
||||
for section_name in ('location', 'storage', 'retention', 'consistency')
|
||||
)
|
||||
remote_path = location.get('remote_path')
|
||||
config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
|
||||
convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
|
||||
|
||||
borg.initialize(storage)
|
||||
if len(config_filenames) == 0:
|
||||
raise ValueError('Error: No configuration files found in: {}'.format(' '.join(args.config_paths)))
|
||||
|
||||
for repository in location['repositories']:
|
||||
borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
|
||||
borg.create_archive(
|
||||
args.verbosity,
|
||||
repository,
|
||||
location,
|
||||
storage,
|
||||
for config_filename in config_filenames:
|
||||
config = validate.parse_configuration(config_filename, validate.schema_filename())
|
||||
(location, storage, retention, consistency) = (
|
||||
config.get(section_name, {})
|
||||
for section_name in ('location', 'storage', 'retention', 'consistency')
|
||||
)
|
||||
borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
|
||||
remote_path = location.get('remote_path')
|
||||
|
||||
borg.initialize(storage)
|
||||
|
||||
for repository in location['repositories']:
|
||||
borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
|
||||
borg.create_archive(
|
||||
args.verbosity,
|
||||
repository,
|
||||
location,
|
||||
storage,
|
||||
)
|
||||
borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
|
||||
except (ValueError, OSError, CalledProcessError) as error:
|
||||
print(error, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
|
27
borgmatic/config/collect.py
Normal file
27
borgmatic/config/collect.py
Normal file
|
@ -0,0 +1,27 @@
|
|||
import os
|
||||
|
||||
|
||||
def collect_config_filenames(config_paths):
|
||||
'''
|
||||
Given a sequence of config paths, both filenames and directories, resolve that to just an
|
||||
iterable of files. Accomplish this by listing any given directories looking for contained config
|
||||
files. This is non-recursive, so any directories within the given directories are ignored.
|
||||
|
||||
Return paths even if they don't exist on disk, so the user can find out about missing
|
||||
configuration paths. However, skip /etc/borgmatic.d if it's missing, so the user doesn't have to
|
||||
create it unless they need it.
|
||||
'''
|
||||
for path in config_paths:
|
||||
exists = os.path.exists(path)
|
||||
|
||||
if os.path.realpath(path) == '/etc/borgmatic.d' and not exists:
|
||||
continue
|
||||
|
||||
if not os.path.isdir(path) or not exists:
|
||||
yield path
|
||||
continue
|
||||
|
||||
for filename in os.listdir(path):
|
||||
full_filename = os.path.join(path, filename)
|
||||
if not os.path.isdir(full_filename):
|
||||
yield full_filename
|
|
@ -77,14 +77,19 @@ instead of the old one.'''
|
|||
)
|
||||
|
||||
|
||||
def guard_configuration_upgraded(source_config_filename, destination_config_filename):
|
||||
def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
|
||||
'''
|
||||
If legacy souce configuration exists but destination upgraded config doesn't, raise
|
||||
If legacy source configuration exists but no destination upgraded configs do, raise
|
||||
LegacyConfigurationNotUpgraded.
|
||||
|
||||
The idea is that we want to alert the user about upgrading their config if they haven't already.
|
||||
'''
|
||||
if os.path.exists(source_config_filename) and not os.path.exists(destination_config_filename):
|
||||
destination_config_exists = any(
|
||||
os.path.exists(filename)
|
||||
for filename in destination_config_filenames
|
||||
)
|
||||
|
||||
if os.path.exists(source_config_filename) and not destination_config_exists:
|
||||
raise LegacyConfigurationNotUpgraded()
|
||||
|
||||
|
||||
|
|
|
@ -9,23 +9,30 @@ from borgmatic.commands import borgmatic as module
|
|||
def test_parse_arguments_with_no_arguments_uses_defaults():
|
||||
parser = module.parse_arguments()
|
||||
|
||||
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
|
||||
assert parser.config_paths == module.DEFAULT_CONFIG_PATHS
|
||||
assert parser.excludes_filename == None
|
||||
assert parser.verbosity is None
|
||||
|
||||
|
||||
def test_parse_arguments_with_filename_arguments_overrides_defaults():
|
||||
def test_parse_arguments_with_path_arguments_overrides_defaults():
|
||||
parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
|
||||
|
||||
assert parser.config_filename == 'myconfig'
|
||||
assert parser.config_paths == ['myconfig']
|
||||
assert parser.excludes_filename == 'myexcludes'
|
||||
assert parser.verbosity is None
|
||||
|
||||
|
||||
def test_parse_arguments_with_multiple_config_paths_parses_as_list():
|
||||
parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
|
||||
|
||||
assert parser.config_paths == ['myconfig', 'otherconfig']
|
||||
assert parser.verbosity is None
|
||||
|
||||
|
||||
def test_parse_arguments_with_verbosity_flag_overrides_default():
|
||||
parser = module.parse_arguments('--verbosity', '1')
|
||||
|
||||
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
|
||||
assert parser.config_paths == module.DEFAULT_CONFIG_PATHS
|
||||
assert parser.excludes_filename == None
|
||||
assert parser.verbosity == 1
|
||||
|
||||
|
|
58
borgmatic/tests/unit/config/test_collect.py
Normal file
58
borgmatic/tests/unit/config/test_collect.py
Normal file
|
@ -0,0 +1,58 @@
|
|||
from flexmock import flexmock
|
||||
|
||||
from borgmatic.config import collect as module
|
||||
|
||||
|
||||
def test_collect_config_filenames_collects_given_files():
|
||||
config_paths = ('config.yaml', 'other.yaml')
|
||||
flexmock(module.os.path).should_receive('isdir').and_return(False)
|
||||
|
||||
config_filenames = tuple(module.collect_config_filenames(config_paths))
|
||||
|
||||
assert config_filenames == config_paths
|
||||
|
||||
|
||||
def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories():
|
||||
config_paths = ('config.yaml', '/etc/borgmatic.d')
|
||||
mock_path = flexmock(module.os.path)
|
||||
mock_path.should_receive('exists').and_return(True)
|
||||
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
|
||||
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
|
||||
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
|
||||
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True)
|
||||
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False)
|
||||
flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar', 'baz.yaml'])
|
||||
|
||||
config_filenames = tuple(module.collect_config_filenames(config_paths))
|
||||
|
||||
assert config_filenames == (
|
||||
'config.yaml',
|
||||
'/etc/borgmatic.d/foo.yaml',
|
||||
'/etc/borgmatic.d/baz.yaml',
|
||||
)
|
||||
|
||||
|
||||
def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist():
|
||||
config_paths = ('config.yaml', '/etc/borgmatic.d')
|
||||
mock_path = flexmock(module.os.path)
|
||||
mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
|
||||
mock_path.should_receive('exists').with_args('/etc/borgmatic.d').and_return(False)
|
||||
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
|
||||
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
|
||||
|
||||
config_filenames = tuple(module.collect_config_filenames(config_paths))
|
||||
|
||||
assert config_filenames == ('config.yaml',)
|
||||
|
||||
|
||||
def test_collect_config_filenames_includes_directory_if_it_does_not_exist():
|
||||
config_paths = ('config.yaml', '/my/directory')
|
||||
mock_path = flexmock(module.os.path)
|
||||
mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
|
||||
mock_path.should_receive('exists').with_args('/my/directory').and_return(False)
|
||||
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
|
||||
mock_path.should_receive('isdir').with_args('/my/directory').and_return(True)
|
||||
|
||||
config_filenames = tuple(module.collect_config_filenames(config_paths))
|
||||
|
||||
assert config_filenames == config_paths
|
|
@ -79,30 +79,34 @@ def test_convert_legacy_parsed_config_splits_space_separated_values():
|
|||
def test_guard_configuration_upgraded_raises_when_only_source_config_present():
|
||||
flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
|
||||
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
|
||||
flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
|
||||
|
||||
with pytest.raises(module.LegacyConfigurationNotUpgraded):
|
||||
module.guard_configuration_upgraded('config', 'config.yaml')
|
||||
module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
|
||||
|
||||
|
||||
def test_guard_configuration_upgraded_does_not_raise_when_only_destination_config_present():
|
||||
flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
|
||||
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True)
|
||||
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
|
||||
flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True)
|
||||
|
||||
module.guard_configuration_upgraded('config', 'config.yaml')
|
||||
module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
|
||||
|
||||
|
||||
def test_guard_configuration_upgraded_does_not_raise_when_both_configs_present():
|
||||
flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
|
||||
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True)
|
||||
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
|
||||
flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True)
|
||||
|
||||
module.guard_configuration_upgraded('config', 'config.yaml')
|
||||
module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
|
||||
|
||||
|
||||
def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present():
|
||||
flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
|
||||
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
|
||||
flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
|
||||
|
||||
module.guard_configuration_upgraded('config', 'config.yaml')
|
||||
module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
|
||||
|
||||
|
||||
def test_guard_excludes_filename_omitted_raises_when_filename_provided():
|
||||
|
|
2
setup.py
2
setup.py
|
@ -1,7 +1,7 @@
|
|||
from setuptools import setup, find_packages
|
||||
|
||||
|
||||
VERSION = '1.1.2'
|
||||
VERSION = '1.1.3.dev0'
|
||||
|
||||
|
||||
setup(
|
||||
|
|
Loading…
Reference in a new issue