Use absolute paths when storing configuration files in an archive for later bootstrapping (#697).

This commit is contained in:
Dan Helfman 2023-06-15 21:45:43 -07:00
parent 9152fed249
commit c294e78715
4 changed files with 55 additions and 58 deletions

View file

@ -81,26 +81,25 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
bootstrap_arguments, global_arguments, local_borg_version bootstrap_arguments, global_arguments, local_borg_version
) )
for config_path in manifest_config_paths: logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
logger.info(f'Bootstrapping config path {config_path}')
borgmatic.borg.extract.extract_archive( borgmatic.borg.extract.extract_archive(
global_arguments.dry_run, global_arguments.dry_run,
bootstrap_arguments.repository,
borgmatic.borg.rlist.resolve_archive_name(
bootstrap_arguments.repository, bootstrap_arguments.repository,
borgmatic.borg.rlist.resolve_archive_name( bootstrap_arguments.archive,
bootstrap_arguments.repository,
bootstrap_arguments.archive,
{},
local_borg_version,
global_arguments,
),
[config_path],
{},
{}, {},
local_borg_version, local_borg_version,
global_arguments, global_arguments,
extract_to_stdout=False, ),
destination_path=bootstrap_arguments.destination, [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
strip_components=bootstrap_arguments.strip_components, {},
progress=bootstrap_arguments.progress, {},
) local_borg_version,
global_arguments,
extract_to_stdout=False,
destination_path=bootstrap_arguments.destination,
strip_components=bootstrap_arguments.strip_components,
progress=bootstrap_arguments.progress,
)

View file

@ -612,26 +612,8 @@ def collect_configuration_run_summary_logs(configs, arguments):
As a side effect of running through these configuration files, output their JSON results, if As a side effect of running through these configuration files, output their JSON results, if
any, to stdout. any, to stdout.
''' '''
# Run cross-file validation checks.
repository = None
for action_name, action_arguments in arguments.items():
if hasattr(action_arguments, 'repository'):
repository = getattr(action_arguments, 'repository')
break
try:
if 'extract' in arguments or 'mount' in arguments:
validate.guard_single_repository_selected(repository, configs)
if 'bootstrap' not in arguments:
validate.guard_configuration_contains_repository(repository, configs)
except ValueError as error:
yield from log_error_records(str(error))
return
if 'bootstrap' in arguments: if 'bootstrap' in arguments:
# no configuration file is needed for bootstrap # No configuration file is needed for bootstrap.
local_borg_version = borg_version.local_borg_version({}, 'borg') local_borg_version = borg_version.local_borg_version({}, 'borg')
try: try:
borgmatic.actions.config.bootstrap.run_bootstrap( borgmatic.actions.config.bootstrap.run_bootstrap(
@ -653,6 +635,23 @@ def collect_configuration_run_summary_logs(configs, arguments):
return return
# Run cross-file validation checks.
repository = None
for action_name, action_arguments in arguments.items():
if hasattr(action_arguments, 'repository'):
repository = getattr(action_arguments, 'repository')
break
try:
if 'extract' in arguments or 'mount' in arguments:
validate.guard_single_repository_selected(repository, configs)
validate.guard_configuration_contains_repository(repository, configs)
except ValueError as error:
yield from log_error_records(str(error))
return
if not configs: if not configs:
yield from log_error_records( yield from log_error_records(
f"{' '.join(arguments['global'].config_paths)}: No valid configuration files found", f"{' '.join(arguments['global'].config_paths)}: No valid configuration files found",

View file

@ -24,9 +24,9 @@ def get_default_config_paths(expand_home=True):
def collect_config_filenames(config_paths): def collect_config_filenames(config_paths):
''' '''
Given a sequence of config paths, both filenames and directories, resolve that to an iterable Given a sequence of config paths, both filenames and directories, resolve that to an iterable
of files. Accomplish this by listing any given directories looking for contained config files of absolute files. Accomplish this by listing any given directories looking for contained config
(ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories within the given files (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories
directories are ignored. within the given directories are ignored.
Return paths even if they don't exist on disk, so the user can find out about missing Return paths even if they don't exist on disk, so the user can find out about missing
configuration paths. However, skip a default config path if it's missing, so the user doesn't configuration paths. However, skip a default config path if it's missing, so the user doesn't
@ -41,7 +41,7 @@ def collect_config_filenames(config_paths):
continue continue
if not os.path.isdir(path) or not exists: if not os.path.isdir(path) or not exists:
yield path yield os.path.abspath(path)
continue continue
if not os.access(path, os.R_OK): if not os.access(path, os.R_OK):
@ -51,4 +51,4 @@ def collect_config_filenames(config_paths):
full_filename = os.path.join(path, filename) full_filename = os.path.join(path, filename)
matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml') matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
if matching_filetype and not os.path.isdir(full_filename): if matching_filetype and not os.path.isdir(full_filename):
yield full_filename yield os.path.abspath(full_filename)

View file

@ -29,15 +29,6 @@ def test_get_default_config_paths_does_not_expand_home_when_false():
assert '$HOME/.config/borgmatic/config.yaml' in config_paths assert '$HOME/.config/borgmatic/config.yaml' in config_paths
def test_collect_config_filenames_collects_given_files():
config_paths = ('config.yaml', 'other.yaml')
flexmock(module.os.path).should_receive('isdir').and_return(False)
config_filenames = tuple(module.collect_config_filenames(config_paths))
assert config_filenames == config_paths
def test_collect_config_filenames_collects_yml_file_endings(): def test_collect_config_filenames_collects_yml_file_endings():
config_paths = ('config.yaml', '/etc/borgmatic.d') config_paths = ('config.yaml', '/etc/borgmatic.d')
mock_path = flexmock(module.os.path) mock_path = flexmock(module.os.path)
@ -45,13 +36,14 @@ def test_collect_config_filenames_collects_yml_file_endings():
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yml').and_return(False)
mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
flexmock(module.os).should_receive('access').and_return(True) flexmock(module.os).should_receive('access').and_return(True)
flexmock(module.os).should_receive('listdir') flexmock(module.os).should_receive('listdir')
flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['foo.yml']) flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['foo.yml'])
config_filenames = tuple(module.collect_config_filenames(config_paths)) config_filenames = tuple(module.collect_config_filenames(config_paths))
assert config_filenames == ('config.yaml', '/etc/borgmatic.d/foo.yml') assert config_filenames == ('/config.yaml', '/etc/borgmatic.d/foo.yml')
def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories(): def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories():
@ -63,6 +55,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False)
mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
flexmock(module.os).should_receive('access').and_return(True) flexmock(module.os).should_receive('access').and_return(True)
flexmock(module.os).should_receive('listdir') flexmock(module.os).should_receive('listdir')
flexmock(sys.modules['builtins']).should_receive('sorted').and_return( flexmock(sys.modules['builtins']).should_receive('sorted').and_return(
@ -72,7 +65,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
config_filenames = tuple(module.collect_config_filenames(config_paths)) config_filenames = tuple(module.collect_config_filenames(config_paths))
assert config_filenames == ( assert config_filenames == (
'config.yaml', '/config.yaml',
'/etc/borgmatic.d/foo.yaml', '/etc/borgmatic.d/foo.yaml',
'/etc/borgmatic.d/baz.yaml', '/etc/borgmatic.d/baz.yaml',
) )
@ -86,6 +79,7 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar.yaml~').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar.yaml~').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.txt').and_return(False) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.txt').and_return(False)
mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
flexmock(module.os).should_receive('access').and_return(True) flexmock(module.os).should_receive('access').and_return(True)
flexmock(module.os).should_receive('listdir') flexmock(module.os).should_receive('listdir')
flexmock(sys.modules['builtins']).should_receive('sorted').and_return( flexmock(sys.modules['builtins']).should_receive('sorted').and_return(
@ -103,13 +97,14 @@ def test_collect_config_filenames_skips_permission_denied_directories():
mock_path.should_receive('exists').and_return(True) mock_path.should_receive('exists').and_return(True)
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
flexmock(module.os).should_receive('access').and_return(False) flexmock(module.os).should_receive('access').and_return(False)
flexmock(module.os).should_receive('listdir') flexmock(module.os).should_receive('listdir')
flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['config.yaml']) flexmock(sys.modules['builtins']).should_receive('sorted').and_return(['config.yaml'])
config_filenames = tuple(module.collect_config_filenames(config_paths)) config_filenames = tuple(module.collect_config_filenames(config_paths))
assert config_filenames == ('config.yaml',) assert config_filenames == ('/config.yaml',)
def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does_not_exist(): def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does_not_exist():
@ -119,10 +114,11 @@ def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does
mock_path.should_receive('exists').with_args('/etc/borgmatic/config.yaml').and_return(False) mock_path.should_receive('exists').with_args('/etc/borgmatic/config.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic/config.yaml').and_return(True) mock_path.should_receive('isdir').with_args('/etc/borgmatic/config.yaml').and_return(True)
mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
config_filenames = tuple(module.collect_config_filenames(config_paths)) config_filenames = tuple(module.collect_config_filenames(config_paths))
assert config_filenames == ('config.yaml',) assert config_filenames == ('/config.yaml',)
def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist(): def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist():
@ -132,10 +128,11 @@ def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist
mock_path.should_receive('exists').with_args('/etc/borgmatic.d').and_return(False) mock_path.should_receive('exists').with_args('/etc/borgmatic.d').and_return(False)
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True)
mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
config_filenames = tuple(module.collect_config_filenames(config_paths)) config_filenames = tuple(module.collect_config_filenames(config_paths))
assert config_filenames == ('config.yaml',) assert config_filenames == ('/config.yaml',)
def test_collect_config_filenames_skips_non_canonical_etc_borgmatic_dot_d_if_it_does_not_exist(): def test_collect_config_filenames_skips_non_canonical_etc_borgmatic_dot_d_if_it_does_not_exist():
@ -145,10 +142,11 @@ def test_collect_config_filenames_skips_non_canonical_etc_borgmatic_dot_d_if_it_
mock_path.should_receive('exists').with_args('/etc/../etc/borgmatic.d').and_return(False) mock_path.should_receive('exists').with_args('/etc/../etc/borgmatic.d').and_return(False)
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/etc/../etc/borgmatic.d').and_return(True) mock_path.should_receive('isdir').with_args('/etc/../etc/borgmatic.d').and_return(True)
mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
config_filenames = tuple(module.collect_config_filenames(config_paths)) config_filenames = tuple(module.collect_config_filenames(config_paths))
assert config_filenames == ('config.yaml',) assert config_filenames == ('/config.yaml',)
def test_collect_config_filenames_includes_other_directory_if_it_does_not_exist(): def test_collect_config_filenames_includes_other_directory_if_it_does_not_exist():
@ -158,7 +156,8 @@ def test_collect_config_filenames_includes_other_directory_if_it_does_not_exist(
mock_path.should_receive('exists').with_args('/my/directory').and_return(False) mock_path.should_receive('exists').with_args('/my/directory').and_return(False)
mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
mock_path.should_receive('isdir').with_args('/my/directory').and_return(True) mock_path.should_receive('isdir').with_args('/my/directory').and_return(True)
mock_path.should_receive('abspath').replace_with(lambda path: module.os.path.join('/', path))
config_filenames = tuple(module.collect_config_filenames(config_paths)) config_filenames = tuple(module.collect_config_filenames(config_paths))
assert config_filenames == config_paths assert config_filenames == ('/config.yaml', '/my/directory')