#14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
This commit is contained in:
parent
57b3066987
commit
7ed5b33db5
9 changed files with 172 additions and 38 deletions
5
NEWS
5
NEWS
|
@ -1,3 +1,8 @@
|
||||||
|
1.1.3.dev0
|
||||||
|
|
||||||
|
* #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run.
|
||||||
|
* Fix for generate-borgmatic-config writing config with invalid one_file_system value.
|
||||||
|
|
||||||
1.1.2
|
1.1.2
|
||||||
|
|
||||||
* #32: Fix for passing check_last as integer to subprocess when calling Borg.
|
* #32: Fix for passing check_last as integer to subprocess when calling Borg.
|
||||||
|
|
23
README.md
23
README.md
|
@ -68,7 +68,9 @@ To install borgmatic, run the following command to download and install it:
|
||||||
Make sure you're using Python 3, as borgmatic does not support Python 2. (You
|
Make sure you're using Python 3, as borgmatic does not support Python 2. (You
|
||||||
may have to use "pip3" or similar instead of "pip".)
|
may have to use "pip3" or similar instead of "pip".)
|
||||||
|
|
||||||
Then, generate a sample configuration file:
|
## Configuration
|
||||||
|
|
||||||
|
After you install borgmatic, generate a sample configuration file:
|
||||||
|
|
||||||
sudo generate-borgmatic-config
|
sudo generate-borgmatic-config
|
||||||
|
|
||||||
|
@ -78,6 +80,25 @@ representative. All fields are optional except where indicated, so feel free
|
||||||
to remove anything you don't need.
|
to remove anything you don't need.
|
||||||
|
|
||||||
|
|
||||||
|
### Multiple configuration files
|
||||||
|
|
||||||
|
A more advanced usage is to create multiple separate configuration files and
|
||||||
|
place each one in a /etc/borgmatic.d directory. For instance:
|
||||||
|
|
||||||
|
sudo mkdir /etc/borgmatic.d
|
||||||
|
sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
|
||||||
|
sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
|
||||||
|
|
||||||
|
With this approach, you can have entirely different backup policies for
|
||||||
|
different applications on your system. For instance, you may want one backup
|
||||||
|
configuration for your database data directory, and a different configuration
|
||||||
|
for your user home directories.
|
||||||
|
|
||||||
|
When you set up multiple configuration files like this, borgmatic will run
|
||||||
|
each one in turn from a single borgmatic invocation. This includes, by
|
||||||
|
default, the traditional /etc/borgmatic/config.yaml as well.
|
||||||
|
|
||||||
|
|
||||||
## Upgrading
|
## Upgrading
|
||||||
|
|
||||||
In general, all you should need to do to upgrade borgmatic is run the
|
In general, all you should need to do to upgrade borgmatic is run the
|
||||||
|
|
|
@ -5,12 +5,12 @@ from subprocess import CalledProcessError
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from borgmatic import borg
|
from borgmatic import borg
|
||||||
from borgmatic.config import convert, validate
|
from borgmatic.config import collect, convert, validate
|
||||||
|
|
||||||
|
|
||||||
LEGACY_CONFIG_FILENAME = '/etc/borgmatic/config'
|
LEGACY_CONFIG_PATH = '/etc/borgmatic/config'
|
||||||
DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
|
DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d']
|
||||||
DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
|
DEFAULT_EXCLUDES_PATH = '/etc/borgmatic/excludes'
|
||||||
|
|
||||||
|
|
||||||
def parse_arguments(*arguments):
|
def parse_arguments(*arguments):
|
||||||
|
@ -21,9 +21,10 @@ def parse_arguments(*arguments):
|
||||||
parser = ArgumentParser()
|
parser = ArgumentParser()
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-c', '--config',
|
'-c', '--config',
|
||||||
dest='config_filename',
|
nargs='+',
|
||||||
default=DEFAULT_CONFIG_FILENAME,
|
dest='config_paths',
|
||||||
help='Configuration filename',
|
default=DEFAULT_CONFIG_PATHS,
|
||||||
|
help='Configuration filenames or directories, defaults to: {}'.format(' '.join(DEFAULT_CONFIG_PATHS)),
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--excludes',
|
'--excludes',
|
||||||
|
@ -42,25 +43,31 @@ def parse_arguments(*arguments):
|
||||||
def main(): # pragma: no cover
|
def main(): # pragma: no cover
|
||||||
try:
|
try:
|
||||||
args = parse_arguments(*sys.argv[1:])
|
args = parse_arguments(*sys.argv[1:])
|
||||||
convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename)
|
config_filenames = tuple(collect.collect_config_filenames(args.config_paths))
|
||||||
config = validate.parse_configuration(args.config_filename, validate.schema_filename())
|
convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
|
||||||
(location, storage, retention, consistency) = (
|
|
||||||
config.get(section_name, {})
|
|
||||||
for section_name in ('location', 'storage', 'retention', 'consistency')
|
|
||||||
)
|
|
||||||
remote_path = location.get('remote_path')
|
|
||||||
|
|
||||||
borg.initialize(storage)
|
if len(config_filenames) == 0:
|
||||||
|
raise ValueError('Error: No configuration files found in: {}'.format(' '.join(args.config_paths)))
|
||||||
|
|
||||||
for repository in location['repositories']:
|
for config_filename in config_filenames:
|
||||||
borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
|
config = validate.parse_configuration(config_filename, validate.schema_filename())
|
||||||
borg.create_archive(
|
(location, storage, retention, consistency) = (
|
||||||
args.verbosity,
|
config.get(section_name, {})
|
||||||
repository,
|
for section_name in ('location', 'storage', 'retention', 'consistency')
|
||||||
location,
|
|
||||||
storage,
|
|
||||||
)
|
)
|
||||||
borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
|
remote_path = location.get('remote_path')
|
||||||
|
|
||||||
|
borg.initialize(storage)
|
||||||
|
|
||||||
|
for repository in location['repositories']:
|
||||||
|
borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path)
|
||||||
|
borg.create_archive(
|
||||||
|
args.verbosity,
|
||||||
|
repository,
|
||||||
|
location,
|
||||||
|
storage,
|
||||||
|
)
|
||||||
|
borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path)
|
||||||
except (ValueError, OSError, CalledProcessError) as error:
|
except (ValueError, OSError, CalledProcessError) as error:
|
||||||
print(error, file=sys.stderr)
|
print(error, file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
27
borgmatic/config/collect.py
Normal file
27
borgmatic/config/collect.py
Normal file
|
@ -0,0 +1,27 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
|
||||||
|
def collect_config_filenames(config_paths):
|
||||||
|
'''
|
||||||
|
Given a sequence of config paths, both filenames and directories, resolve that to just an
|
||||||
|
iterable of files. Accomplish this by listing any given directories looking for contained config
|
||||||
|
files. This is non-recursive, so any directories within the given directories are ignored.
|
||||||
|
|
||||||
|
Return paths even if they don't exist on disk, so the user can find out about missing
|
||||||
|
configuration paths. However, skip /etc/borgmatic.d if it's missing, so the user doesn't have to
|
||||||
|
create it unless they need it.
|
||||||
|
'''
|
||||||
|
for path in config_paths:
|
||||||
|
exists = os.path.exists(path)
|
||||||
|
|
||||||
|
if os.path.realpath(path) == '/etc/borgmatic.d' and not exists:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not os.path.isdir(path) or not exists:
|
||||||
|
yield path
|
||||||
|
continue
|
||||||
|
|
||||||
|
for filename in os.listdir(path):
|
||||||
|
full_filename = os.path.join(path, filename)
|
||||||
|
if not os.path.isdir(full_filename):
|
||||||
|
yield full_filename
|
|
@ -77,14 +77,19 @@ instead of the old one.'''
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def guard_configuration_upgraded(source_config_filename, destination_config_filename):
|
def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
|
||||||
'''
|
'''
|
||||||
If legacy souce configuration exists but destination upgraded config doesn't, raise
|
If legacy source configuration exists but no destination upgraded configs do, raise
|
||||||
LegacyConfigurationNotUpgraded.
|
LegacyConfigurationNotUpgraded.
|
||||||
|
|
||||||
The idea is that we want to alert the user about upgrading their config if they haven't already.
|
The idea is that we want to alert the user about upgrading their config if they haven't already.
|
||||||
'''
|
'''
|
||||||
if os.path.exists(source_config_filename) and not os.path.exists(destination_config_filename):
|
destination_config_exists = any(
|
||||||
|
os.path.exists(filename)
|
||||||
|
for filename in destination_config_filenames
|
||||||
|
)
|
||||||
|
|
||||||
|
if os.path.exists(source_config_filename) and not destination_config_exists:
|
||||||
raise LegacyConfigurationNotUpgraded()
|
raise LegacyConfigurationNotUpgraded()
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -9,23 +9,30 @@ from borgmatic.commands import borgmatic as module
|
||||||
def test_parse_arguments_with_no_arguments_uses_defaults():
|
def test_parse_arguments_with_no_arguments_uses_defaults():
|
||||||
parser = module.parse_arguments()
|
parser = module.parse_arguments()
|
||||||
|
|
||||||
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
|
assert parser.config_paths == module.DEFAULT_CONFIG_PATHS
|
||||||
assert parser.excludes_filename == None
|
assert parser.excludes_filename == None
|
||||||
assert parser.verbosity is None
|
assert parser.verbosity is None
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_filename_arguments_overrides_defaults():
|
def test_parse_arguments_with_path_arguments_overrides_defaults():
|
||||||
parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
|
parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
|
||||||
|
|
||||||
assert parser.config_filename == 'myconfig'
|
assert parser.config_paths == ['myconfig']
|
||||||
assert parser.excludes_filename == 'myexcludes'
|
assert parser.excludes_filename == 'myexcludes'
|
||||||
assert parser.verbosity is None
|
assert parser.verbosity is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_arguments_with_multiple_config_paths_parses_as_list():
|
||||||
|
parser = module.parse_arguments('--config', 'myconfig', 'otherconfig')
|
||||||
|
|
||||||
|
assert parser.config_paths == ['myconfig', 'otherconfig']
|
||||||
|
assert parser.verbosity is None
|
||||||
|
|
||||||
|
|
||||||
def test_parse_arguments_with_verbosity_flag_overrides_default():
|
def test_parse_arguments_with_verbosity_flag_overrides_default():
|
||||||
parser = module.parse_arguments('--verbosity', '1')
|
parser = module.parse_arguments('--verbosity', '1')
|
||||||
|
|
||||||
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
|
assert parser.config_paths == module.DEFAULT_CONFIG_PATHS
|
||||||
assert parser.excludes_filename == None
|
assert parser.excludes_filename == None
|
||||||
assert parser.verbosity == 1
|
assert parser.verbosity == 1
|
||||||
|
|
||||||
|
|
58
borgmatic/tests/unit/config/test_collect.py
Normal file
58
borgmatic/tests/unit/config/test_collect.py
Normal file
|
@ -0,0 +1,58 @@
|
||||||
|
from flexmock import flexmock
|
||||||
|
|
||||||
|
from borgmatic.config import collect as module
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_config_filenames_collects_given_files():
|
||||||
|
config_paths = ('config.yaml', 'other.yaml')
|
||||||
|
flexmock(module.os.path).should_receive('isdir').and_return(False)
|
||||||
|
|
||||||
|
config_filenames = tuple(module.collect_config_filenames(config_paths))
|
||||||
|
|
||||||
|
assert config_filenames == config_paths
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories():
|
||||||
|
config_paths = ('config.yaml', '/etc/borgmatic.d')
|
||||||
|
mock_path = flexmock(module.os.path)
|
||||||
|
mock_path.should_receive('exists').and_return(True)
|
||||||
|
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
|
||||||
|
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
|
||||||
|
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
|
||||||
|
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True)
|
||||||
|
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False)
|
||||||
|
flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar', 'baz.yaml'])
|
||||||
|
|
||||||
|
config_filenames = tuple(module.collect_config_filenames(config_paths))
|
||||||
|
|
||||||
|
assert config_filenames == (
|
||||||
|
'config.yaml',
|
||||||
|
'/etc/borgmatic.d/foo.yaml',
|
||||||
|
'/etc/borgmatic.d/baz.yaml',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist():
|
||||||
|
config_paths = ('config.yaml', '/etc/borgmatic.d')
|
||||||
|
mock_path = flexmock(module.os.path)
|
||||||
|
mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
|
||||||
|
mock_path.should_receive('exists').with_args('/etc/borgmatic.d').and_return(False)
|
||||||
|
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
|
||||||
|
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
|
||||||
|
|
||||||
|
config_filenames = tuple(module.collect_config_filenames(config_paths))
|
||||||
|
|
||||||
|
assert config_filenames == ('config.yaml',)
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_config_filenames_includes_directory_if_it_does_not_exist():
|
||||||
|
config_paths = ('config.yaml', '/my/directory')
|
||||||
|
mock_path = flexmock(module.os.path)
|
||||||
|
mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
|
||||||
|
mock_path.should_receive('exists').with_args('/my/directory').and_return(False)
|
||||||
|
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
|
||||||
|
mock_path.should_receive('isdir').with_args('/my/directory').and_return(True)
|
||||||
|
|
||||||
|
config_filenames = tuple(module.collect_config_filenames(config_paths))
|
||||||
|
|
||||||
|
assert config_filenames == config_paths
|
|
@ -79,30 +79,34 @@ def test_convert_legacy_parsed_config_splits_space_separated_values():
|
||||||
def test_guard_configuration_upgraded_raises_when_only_source_config_present():
|
def test_guard_configuration_upgraded_raises_when_only_source_config_present():
|
||||||
flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
|
flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
|
||||||
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
|
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
|
||||||
|
flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
|
||||||
|
|
||||||
with pytest.raises(module.LegacyConfigurationNotUpgraded):
|
with pytest.raises(module.LegacyConfigurationNotUpgraded):
|
||||||
module.guard_configuration_upgraded('config', 'config.yaml')
|
module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
|
||||||
|
|
||||||
|
|
||||||
def test_guard_configuration_upgraded_does_not_raise_when_only_destination_config_present():
|
def test_guard_configuration_upgraded_does_not_raise_when_only_destination_config_present():
|
||||||
flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
|
flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
|
||||||
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True)
|
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
|
||||||
|
flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True)
|
||||||
|
|
||||||
module.guard_configuration_upgraded('config', 'config.yaml')
|
module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
|
||||||
|
|
||||||
|
|
||||||
def test_guard_configuration_upgraded_does_not_raise_when_both_configs_present():
|
def test_guard_configuration_upgraded_does_not_raise_when_both_configs_present():
|
||||||
flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
|
flexmock(os.path).should_receive('exists').with_args('config').and_return(True)
|
||||||
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True)
|
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
|
||||||
|
flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True)
|
||||||
|
|
||||||
module.guard_configuration_upgraded('config', 'config.yaml')
|
module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
|
||||||
|
|
||||||
|
|
||||||
def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present():
|
def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present():
|
||||||
flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
|
flexmock(os.path).should_receive('exists').with_args('config').and_return(False)
|
||||||
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
|
flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False)
|
||||||
|
flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False)
|
||||||
|
|
||||||
module.guard_configuration_upgraded('config', 'config.yaml')
|
module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml'))
|
||||||
|
|
||||||
|
|
||||||
def test_guard_excludes_filename_omitted_raises_when_filename_provided():
|
def test_guard_excludes_filename_omitted_raises_when_filename_provided():
|
||||||
|
|
2
setup.py
2
setup.py
|
@ -1,7 +1,7 @@
|
||||||
from setuptools import setup, find_packages
|
from setuptools import setup, find_packages
|
||||||
|
|
||||||
|
|
||||||
VERSION = '1.1.2'
|
VERSION = '1.1.3.dev0'
|
||||||
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
|
|
Loading…
Reference in a new issue