feat: tag repos

This commit is contained in:
Divyansh Singh 2023-03-23 01:01:26 +05:30
parent f9ef52f9a5
commit 8a63c49498
5 changed files with 69 additions and 35 deletions

View file

@ -42,11 +42,11 @@ def run_create(
global_arguments.dry_run, global_arguments.dry_run,
**hook_context, **hook_context,
) )
logger.info('{}: Creating archive{}'.format(repository, dry_run_label)) logger.info('{}: Creating archive{}'.format(repository['path'], dry_run_label))
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
'remove_database_dumps', 'remove_database_dumps',
hooks, hooks,
repository, repository['path'],
borgmatic.hooks.dump.DATABASE_HOOK_NAMES, borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location, location,
global_arguments.dry_run, global_arguments.dry_run,
@ -54,7 +54,7 @@ def run_create(
active_dumps = borgmatic.hooks.dispatch.call_hooks( active_dumps = borgmatic.hooks.dispatch.call_hooks(
'dump_databases', 'dump_databases',
hooks, hooks,
repository, repository['path'],
borgmatic.hooks.dump.DATABASE_HOOK_NAMES, borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
location, location,
global_arguments.dry_run, global_arguments.dry_run,
@ -63,7 +63,7 @@ def run_create(
json_output = borgmatic.borg.create.create_archive( json_output = borgmatic.borg.create.create_archive(
global_arguments.dry_run, global_arguments.dry_run,
repository, repository['path'],
location, location,
storage, storage,
local_borg_version, local_borg_version,

View file

@ -108,7 +108,7 @@ def run_configuration(config_filename, config, arguments):
repo_queue.put((repo, 0),) repo_queue.put((repo, 0),)
while not repo_queue.empty(): while not repo_queue.empty():
repository_path, retry_num = repo_queue.get() repository, retry_num = repo_queue.get()
timeout = retry_num * retry_wait timeout = retry_num * retry_wait
if timeout: if timeout:
logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry')
@ -125,14 +125,14 @@ def run_configuration(config_filename, config, arguments):
local_path=local_path, local_path=local_path,
remote_path=remote_path, remote_path=remote_path,
local_borg_version=local_borg_version, local_borg_version=local_borg_version,
repository_path=repository_path, repository=repository,
) )
except (OSError, CalledProcessError, ValueError) as error: except (OSError, CalledProcessError, ValueError) as error:
if retry_num < retries: if retry_num < retries:
repo_queue.put((repository_path, retry_num + 1),) repo_queue.put((repository, retry_num + 1),)
tuple( # Consume the generator so as to trigger logging. tuple( # Consume the generator so as to trigger logging.
log_error_records( log_error_records(
'{}: Error running actions for repository'.format(repository_path), '{}: Error running actions for repository'.format(repository['path']),
error, error,
levelno=logging.WARNING, levelno=logging.WARNING,
log_command_error_output=True, log_command_error_output=True,
@ -147,10 +147,10 @@ def run_configuration(config_filename, config, arguments):
return return
yield from log_error_records( yield from log_error_records(
'{}: Error running actions for repository'.format(repository_path), error '{}: Error running actions for repository'.format(repository['path']), error
) )
encountered_error = error encountered_error = error
error_repository = repository_path error_repository = repository['path']
try: try:
if using_primary_action: if using_primary_action:
@ -248,7 +248,7 @@ def run_actions(
local_path, local_path,
remote_path, remote_path,
local_borg_version, local_borg_version,
repository_path, repository,
): ):
''' '''
Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration
@ -263,13 +263,13 @@ def run_actions(
invalid. invalid.
''' '''
add_custom_log_levels() add_custom_log_levels()
repository = os.path.expanduser(repository_path) repository_path = os.path.expanduser(repository['path'])
global_arguments = arguments['global'] global_arguments = arguments['global']
dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else '' dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else ''
hook_context = { hook_context = {
'repository': repository_path, 'repository': repository_path,
# Deprecated: For backwards compatibility with borgmatic < 1.6.0. # Deprecated: For backwards compatibility with borgmatic < 1.6.0.
'repositories': ','.join(location['repositories']), 'repositories': ','.join([repo['path'] for repo in location['repositories']]),
} }
command.execute_hook( command.execute_hook(

View file

@ -56,9 +56,13 @@ def normalize(config_filename, config):
# Upgrade remote repositories to ssh:// syntax, required in Borg 2. # Upgrade remote repositories to ssh:// syntax, required in Borg 2.
repositories = location.get('repositories') repositories = location.get('repositories')
if isinstance(repositories[0], str):
config['location']['repositories'] = [{'path': repository} for repository in repositories]
repositories = config['location']['repositories']
if repositories: if repositories:
config['location']['repositories'] = [] config['location']['repositories'] = []
for repository in repositories: for repository_dict in repositories:
repository = repository_dict['path']
if '~' in repository: if '~' in repository:
logs.append( logs.append(
logging.makeLogRecord( logging.makeLogRecord(
@ -71,11 +75,19 @@ def normalize(config_filename, config):
) )
if ':' in repository: if ':' in repository:
if repository.startswith('file://'): if repository.startswith('file://'):
updated_repository_path = os.path.abspath(repository.partition('file://')[-1])
config['location']['repositories'].append( config['location']['repositories'].append(
os.path.abspath(repository.partition('file://')[-1]) {
'path': updated_repository_path,
'label': repository_dict.get('label', None),
}
) )
elif repository.startswith('ssh://'): elif repository.startswith('ssh://'):
config['location']['repositories'].append(repository) config['location']['repositories'].append({
'path': repository,
'label': repository_dict.get('label', None),
})
else: else:
rewritten_repository = f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}" rewritten_repository = f"ssh://{repository.replace(':~', '/~').replace(':/', '/').replace(':', '/./')}"
logs.append( logs.append(
@ -87,8 +99,16 @@ def normalize(config_filename, config):
) )
) )
) )
config['location']['repositories'].append(rewritten_repository) config['location']['repositories'].append({
'path': rewritten_repository,
'label': repository_dict.get('label', None),
})
else: else:
config['location']['repositories'].append(repository) config['location']['repositories'].append(
{
'path': repository,
'label': repository_dict.get('label', None),
}
)
return logs return logs

View file

@ -29,19 +29,30 @@ properties:
repositories: repositories:
type: array type: array
items: items:
type: string type: object
description: | required:
Paths to local or remote repositories (required). Tildes are - path
expanded. Multiple repositories are backed up to in properties:
sequence. Borg placeholders can be used. See the output of path:
"borg help placeholders" for details. See ssh_command for type: string
SSH options like identity file or port. If systemd service description: |
is used, then add local repository paths in the systemd Path to local or remote repository (required).
service file to the ReadWritePaths list. are expanded. Multiple repositories are backed up to
example: in sequence. Borg placeholders can be used. See the
- ssh://user@backupserver/./sourcehostname.borg output of "borg help placeholders" for details. See
- ssh://user@backupserver/./{fqdn} ssh_command for SSH options like identity file or
- /var/local/backups/local.borg port. If systemd service is used, then add local
repository paths in the systemd service file to the
ReadWritePaths list.
example:
- ssh://user@backupserver/./sourcehostname.borg
- ssh://user@backupserver/./{fqdn}
- /var/local/backups/local.borg
label:
type: string
description: |
Optional label for the repository.
example: backupserver
working_directory: working_directory:
type: string type: string
description: | description: |

View file

@ -138,10 +138,13 @@ def normalize_repository_path(repository):
def repositories_match(first, second): def repositories_match(first, second):
''' '''
Given two repository paths (relative and/or absolute), return whether they match. Given two repository dicts with keys 'path' (relative and/or absolute),
and 'label', return whether they match.
''' '''
return normalize_repository_path(first) == normalize_repository_path(second) if isinstance(first,str) and isinstance(second,str):
return normalize_repository_path(first) == normalize_repository_path(second)
elif isinstance(first,dict) and isinstance(second,str):
return (second == first.get('label')) or (normalize_repository_path(second) == normalize_repository_path(first.get('path')))
def guard_configuration_contains_repository(repository, configurations): def guard_configuration_contains_repository(repository, configurations):
''' '''
@ -160,7 +163,7 @@ def guard_configuration_contains_repository(repository, configurations):
config_repository config_repository
for config in configurations.values() for config in configurations.values()
for config_repository in config['location']['repositories'] for config_repository in config['location']['repositories']
if repositories_match(repository, config_repository) if repositories_match(config_repository, repository)
) )
) )