psw/borgmatic
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove optional label for repos from tests

Browse source
This commit is contained in:
Divyansh Singh 2023-03-26 16:59:29 +05:30
parent d6dfb8753a
commit b511e679ae
7 changed files with 86 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
borgmatic/commands/borgmatic.py
View file

@ -107,8 +107,7 @@ def run_configuration(config_filename, config, arguments):
while not repo_queue.empty():
repository, retry_num = repo_queue.get()
if isinstance(repository, str):
repository = {'path': repository}
logger.debug(f'{repository["path"]}: Running actions for repository')
timeout = retry_num * retry_wait
if timeout:
logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry')
@ -261,8 +260,6 @@ def run_actions(
invalid.
'''
add_custom_log_levels()
if isinstance(repository, str):
repository = {'path': repository}
repository_path = os.path.expanduser(repository['path'])
global_arguments = arguments['global']
dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else ''

19
borgmatic/config/normalize.py
View file

@ -80,17 +80,11 @@ def normalize(config_filename, config):
updated_repository_path = os.path.abspath(
repository_path.partition('file://')[-1]
)
config['location']['repositories'].append(
{
'path': updated_repository_path,
'label': repository_dict.get('label', ''),
}
dict(repository_dict, path=updated_repository_path,)
)
elif repository_path.startswith('ssh://'):
config['location']['repositories'].append(
{'path': repository_path, 'label': repository_dict.get('label', '')}
)
config['location']['repositories'].append(repository_dict)
else:
rewritten_repository_path = f"ssh://{repository_path.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}"
logs.append(
@ -103,14 +97,9 @@ def normalize(config_filename, config):
)
)
config['location']['repositories'].append(
{
'path': rewritten_repository_path,
'label': repository_dict.get('label', ''),
}
dict(repository_dict, path=rewritten_repository_path,)
)
else:
config['location']['repositories'].append(
{'path': repository_path, 'label': repository_dict.get('label', '')}
)
config['location']['repositories'].append(repository_dict)
return logs

2
borgmatic/config/schema.yaml
View file

@ -53,7 +53,7 @@ properties:
type: string
description: |
Optional label for the repository. This
can be used with the --repository option
can be used with the --repository flag
to select a repository to backup to.
If not specified, the repository path is
used as the label.

2
borgmatic/config/validate.py
View file

@ -138,7 +138,7 @@ def normalize_repository_path(repository):
def repositories_match(first, second):
'''
Given two repository dicts with keys 'path' (relative and/or absolute),
and 'label', return whether they match.
and 'label', or two repository paths, return whether they match.
'''
if isinstance(first, str):
first = {'path': first, 'label': first}

18
tests/integration/config/test_validate.py
View file

@ -65,7 +65,7 @@ def test_parse_configuration_transforms_file_into_mapping():
assert config == {
'location': {
'source_directories': ['/home', '/etc'],
'repositories': [{'path': 'hostname.borg', 'label': ''}],
'repositories': [{'path': 'hostname.borg'}],
},
'retention': {'keep_daily': 7, 'keep_hourly': 24, 'keep_minutely': 60},
'consistency': {'checks': [{'name': 'repository'}, {'name': 'archives'}]},
@ -92,7 +92,7 @@ def test_parse_configuration_passes_through_quoted_punctuation():
assert config == {
'location': {
'source_directories': [f'/home/{string.punctuation}'],
'repositories': [{'path': 'test.borg', 'label': ''}],
'repositories': [{'path': 'test.borg'}],
}
}
assert logs == []
@ -154,10 +154,7 @@ def test_parse_configuration_inlines_include():
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
assert config == {
'location': {
'source_directories': ['/home'],
'repositories': [{'path': 'hostname.borg', 'label': ''}],
},
'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]},
'retention': {'keep_daily': 7, 'keep_hourly': 24},
}
assert logs == []
@ -191,10 +188,7 @@ def test_parse_configuration_merges_include():
config, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml')
assert config == {
'location': {
'source_directories': ['/home'],
'repositories': [{'path': 'hostname.borg', 'label': ''}],
},
'location': {'source_directories': ['/home'], 'repositories': [{'path': 'hostname.borg'}]},
'retention': {'keep_daily': 1, 'keep_hourly': 24},
}
assert logs == []
@ -256,7 +250,7 @@ def test_parse_configuration_applies_overrides():
assert config == {
'location': {
'source_directories': ['/home'],
'repositories': [{'path': 'hostname.borg', 'label': ''}],
'repositories': [{'path': 'hostname.borg'}],
'local_path': 'borg2',
}
}
@ -282,7 +276,7 @@ def test_parse_configuration_applies_normalization():
assert config == {
'location': {
'source_directories': ['/home'],
'repositories': [{'path': 'hostname.borg', 'label': ''}],
'repositories': [{'path': 'hostname.borg'}],
'exclude_if_present': ['.nobackup'],
}
}

130
tests/unit/commands/test_borgmatic.py
View file

@ -15,7 +15,7 @@ def test_run_configuration_runs_actions_for_each_repository():
flexmock(module).should_receive('run_actions').and_return(expected_results[:1]).and_return(
expected_results[1:]
)
config = {'location': {'repositories': ['foo', 'bar']}}
config = {'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}}
arguments = {'global': flexmock(monitoring_verbosity=1)}
results = list(module.run_configuration('test.yaml', config, arguments))
@ -75,7 +75,7 @@ def test_run_configuration_logs_actions_error():
expected_results = [flexmock()]
flexmock(module).should_receive('log_error_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').and_raise(OSError)
config = {'location': {'repositories': ['foo']}}
config = {'location': {'repositories': [{'path': 'foo'}]}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)}
results = list(module.run_configuration('test.yaml', config, arguments))
@ -91,7 +91,7 @@ def test_run_configuration_bails_for_actions_soft_failure():
flexmock(module).should_receive('run_actions').and_raise(error)
flexmock(module).should_receive('log_error_records').never()
flexmock(module.command).should_receive('considered_soft_failure').and_return(True)
config = {'location': {'repositories': ['foo']}}
config = {'location': {'repositories': [{'path': 'foo'}]}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
@ -108,7 +108,7 @@ def test_run_configuration_logs_monitor_log_error():
expected_results = [flexmock()]
flexmock(module).should_receive('log_error_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').and_return([])
config = {'location': {'repositories': ['foo']}}
config = {'location': {'repositories': [{'path': 'foo'}]}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
@ -126,7 +126,7 @@ def test_run_configuration_bails_for_monitor_log_soft_failure():
flexmock(module).should_receive('log_error_records').never()
flexmock(module).should_receive('run_actions').and_return([])
flexmock(module.command).should_receive('considered_soft_failure').and_return(True)
config = {'location': {'repositories': ['foo']}}
config = {'location': {'repositories': [{'path': 'foo'}]}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
@ -143,7 +143,7 @@ def test_run_configuration_logs_monitor_finish_error():
expected_results = [flexmock()]
flexmock(module).should_receive('log_error_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').and_return([])
config = {'location': {'repositories': ['foo']}}
config = {'location': {'repositories': [{'path': 'foo'}]}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
@ -161,7 +161,7 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure():
flexmock(module).should_receive('log_error_records').never()
flexmock(module).should_receive('run_actions').and_return([])
flexmock(module.command).should_receive('considered_soft_failure').and_return(True)
config = {'location': {'repositories': ['foo']}}
config = {'location': {'repositories': [{'path': 'foo'}]}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
@ -178,7 +178,7 @@ def test_run_configuration_logs_on_error_hook_error():
expected_results[:1]
).and_return(expected_results[1:])
flexmock(module).should_receive('run_actions').and_raise(OSError)
config = {'location': {'repositories': ['foo']}}
config = {'location': {'repositories': [{'path': 'foo'}]}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
@ -194,7 +194,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure():
expected_results = [flexmock()]
flexmock(module).should_receive('log_error_records').and_return(expected_results)
flexmock(module).should_receive('run_actions').and_raise(OSError)
config = {'location': {'repositories': ['foo']}}
config = {'location': {'repositories': [{'path': 'foo'}]}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
@ -209,7 +209,7 @@ def test_run_configuration_retries_soft_error():
flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([])
flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once()
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}}
config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == []
@ -231,7 +231,7 @@ def test_run_configuration_retries_hard_error():
flexmock(module).should_receive('log_error_records').with_args(
'foo: Error running actions for repository', OSError,
).and_return(error_logs)
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}}
config = {'location': {'repositories': [{'path': 'foo'}]}, 'storage': {'retries': 1}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == error_logs
@ -249,41 +249,41 @@ def test_run_configuration_repos_ordered():
flexmock(module).should_receive('log_error_records').with_args(
'bar: Error running actions for repository', OSError
).and_return(expected_results[1:]).ordered()
config = {'location': {'repositories': ['foo', 'bar']}}
config = {'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == expected_results
def test_run_configuration_retries_round_robbin():
flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO)
flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
flexmock(module.command).should_receive('execute_hook')
flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
flexmock(module).should_receive('log_error_records').with_args(
'foo: Error running actions for repository',
OSError,
levelno=logging.WARNING,
log_command_error_output=True,
).and_return([flexmock()]).ordered()
flexmock(module).should_receive('log_error_records').with_args(
'bar: Error running actions for repository',
OSError,
levelno=logging.WARNING,
log_command_error_output=True,
).and_return([flexmock()]).ordered()
foo_error_logs = [flexmock()]
flexmock(module).should_receive('log_error_records').with_args(
'foo: Error running actions for repository', OSError
).and_return(foo_error_logs).ordered()
bar_error_logs = [flexmock()]
flexmock(module).should_receive('log_error_records').with_args(
'bar: Error running actions for repository', OSError
).and_return(bar_error_logs).ordered()
config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == foo_error_logs + bar_error_logs
# def test_run_configuration_retries_round_robin():
# flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO)
# flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock())
# flexmock(module.command).should_receive('execute_hook')
# flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
# flexmock(module).should_receive('log_error_records').with_args(
# 'foo: Error running actions for repository',
# OSError,
# levelno=logging.WARNING,
# log_command_error_output=True,
# ).and_return([flexmock()]).ordered()
# flexmock(module).should_receive('log_error_records').with_args(
# 'bar: Error running actions for repository',
# OSError,
# levelno=logging.WARNING,
# log_command_error_output=True,
# ).and_return([flexmock()]).ordered()
# foo_error_logs = [flexmock()]
# flexmock(module).should_receive('log_error_records').with_args(
# 'foo: Error running actions for repository', OSError
# ).and_return(foo_error_logs).ordered()
# bar_error_logs = [flexmock()]
# flexmock(module).should_receive('log_error_records').with_args(
# 'bar: Error running actions for repository', OSError
# ).and_return(bar_error_logs).ordered()
# config = {'location': {'repositories': [{'path':'foo','path':'bar'}]}, 'storage': {'retries': 1}}
# arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
# results = list(module.run_configuration('test.yaml', config, arguments))
# assert results == foo_error_logs + bar_error_logs
def test_run_configuration_retries_one_passes():
@ -309,7 +309,10 @@ def test_run_configuration_retries_one_passes():
flexmock(module).should_receive('log_error_records').with_args(
'bar: Error running actions for repository', OSError
).and_return(error_logs).ordered()
config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
config = {
'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]},
'storage': {'retries': 1},
}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == error_logs
@ -348,7 +351,10 @@ def test_run_configuration_retry_wait():
flexmock(module).should_receive('log_error_records').with_args(
'foo: Error running actions for repository', OSError
).and_return(error_logs).ordered()
config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_wait': 10}}
config = {
'location': {'repositories': [{'path': 'foo'}]},
'storage': {'retries': 3, 'retry_wait': 10},
}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
results = list(module.run_configuration('test.yaml', config, arguments))
assert results == error_logs
@ -384,7 +390,7 @@ def test_run_configuration_retries_timeout_multiple_repos():
'bar: Error running actions for repository', OSError
).and_return(error_logs).ordered()
config = {
'location': {'repositories': ['foo', 'bar']},
'location': {'repositories': [{'path': 'foo'}, {'path': 'bar'}]},
'storage': {'retries': 1, 'retry_wait': 10},
}
arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
@ -409,7 +415,7 @@ def test_run_actions_runs_rcreate():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -431,7 +437,7 @@ def test_run_actions_runs_transfer():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -454,7 +460,7 @@ def test_run_actions_runs_create():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
assert result == (expected,)
@ -477,7 +483,7 @@ def test_run_actions_runs_prune():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -499,7 +505,7 @@ def test_run_actions_runs_compact():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -522,7 +528,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -545,7 +551,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -567,7 +573,7 @@ def test_run_actions_runs_extract():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -589,7 +595,7 @@ def test_run_actions_runs_export_tar():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -611,7 +617,7 @@ def test_run_actions_runs_mount():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -633,7 +639,7 @@ def test_run_actions_runs_restore():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -656,7 +662,7 @@ def test_run_actions_runs_rlist():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
assert result == (expected,)
@ -680,7 +686,7 @@ def test_run_actions_runs_list():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
assert result == (expected,)
@ -704,7 +710,7 @@ def test_run_actions_runs_rinfo():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
assert result == (expected,)
@ -728,7 +734,7 @@ def test_run_actions_runs_info():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
assert result == (expected,)
@ -751,7 +757,7 @@ def test_run_actions_runs_break_lock():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -773,7 +779,7 @@ def test_run_actions_runs_borg():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)
@ -800,7 +806,7 @@ def test_run_actions_runs_multiple_actions_in_argument_order():
local_path=flexmock(),
remote_path=flexmock(),
local_borg_version=flexmock(),
repository='repo',
repository={'path': 'repo'},
)
)

10
tests/unit/config/test_normalize.py
View file

@ -69,27 +69,27 @@ from borgmatic.config import normalize as module
),
(
{'location': {'repositories': ['foo@bar:/repo']}},
{'location': {'repositories': [{'path': 'ssh://foo@bar/repo', 'label': ''}]}},
{'location': {'repositories': [{'path': 'ssh://foo@bar/repo'}]}},
True,
),
(
{'location': {'repositories': ['foo@bar:repo']}},
{'location': {'repositories': [{'path': 'ssh://foo@bar/./repo', 'label': ''}]}},
{'location': {'repositories': [{'path': 'ssh://foo@bar/./repo'}]}},
True,
),
(
{'location': {'repositories': ['foo@bar:~/repo']}},
{'location': {'repositories': [{'path': 'ssh://foo@bar/~/repo', 'label': ''}]}},
{'location': {'repositories': [{'path': 'ssh://foo@bar/~/repo'}]}},
True,
),
(
{'location': {'repositories': ['ssh://foo@bar:1234/repo']}},
{'location': {'repositories': [{'path': 'ssh://foo@bar:1234/repo', 'label': ''}]}},
{'location': {'repositories': [{'path': 'ssh://foo@bar:1234/repo'}]}},
False,
),
(
{'location': {'repositories': ['file:///repo']}},
{'location': {'repositories': [{'path': '/repo', 'label': ''}]}},
{'location': {'repositories': [{'path': '/repo'}]}},
False,
),
),