diff --git a/NEWS b/NEWS index 2021467..9308507 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,7 @@ -1.7.13.dev0 +1.7.14.dev0 + * #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist. + +1.7.13 * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema" flag. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas @@ -7,9 +10,22 @@ commands with arguments. * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break database dumping. + * #680: Add support for logging each log line as a JSON object via global "--log-json" flag. * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories. * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to update your development checkouts accordingly. + * #686: Add fish shell completion script so you can tab-complete on the borgmatic command-line. See + the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion + * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install + --editable" development installs. + * #688: Fix archive checks being skipped even when particular archives haven't been checked + recently. This occurred when using multiple borgmatic configuration files with different + "archive_name_format"s, for instance. + * #691: Fix error in "borgmatic restore" action when the configured repository path is relative + instead of absolute. + * #694: Run "borgmatic borg" action without capturing output so interactive prompts and flags like + "--progress" still work. 1.7.12 * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file. diff --git a/README.md b/README.md index 2af792b..079ec8a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ borgmatic is simple, configuration-driven backup software for servers and workstations. Protect your files with client-side encryption. Backup your databases too. Monitor it all with integrated third-party services. -The canonical home of borgmatic is at https://torsion.org/borgmatic. +The canonical home of borgmatic is at https://torsion.org/borgmatic/ Here's an example configuration file: diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index 3d2998b..44ffc95 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -12,6 +12,7 @@ def run_borg( storage, local_borg_version, borg_arguments, + global_arguments, local_path, remote_path, ): @@ -21,12 +22,15 @@ def run_borg( if borg_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, borg_arguments.repository ): - logger.info(f'{repository["path"]}: Running arbitrary Borg command') + logger.info( + f'{repository.get("label", repository["path"])}: Running arbitrary Borg command' + ) archive_name = borgmatic.borg.rlist.resolve_archive_name( repository['path'], borg_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index 2174161..a00d578 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -11,6 +11,7 @@ def run_break_lock( storage, local_borg_version, break_lock_arguments, + global_arguments, local_path, remote_path, ): @@ -20,11 +21,14 @@ def run_break_lock( if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, break_lock_arguments.repository ): - logger.info(f'{repository["path"]}: Breaking repository and cache locks') + logger.info( + f'{repository.get("label", repository["path"])}: Breaking repository and cache locks' + ) borgmatic.borg.break_lock.break_lock( repository['path'], storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 1696e07..610d41e 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -37,13 +37,14 @@ def run_check( global_arguments.dry_run, **hook_context, ) - logger.info(f'{repository["path"]}: Running consistency checks') + logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks') borgmatic.borg.check.check_archives( repository['path'], location, storage, consistency, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, progress=check_arguments.progress, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 95334c5..ad680d2 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -39,12 +39,15 @@ def run_compact( **hook_context, ) if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): - logger.info(f'{repository["path"]}: Compacting segments{dry_run_label}') + logger.info( + f'{repository.get("label", repository["path"])}: Compacting segments{dry_run_label}' + ) borgmatic.borg.compact.compact_segments( global_arguments.dry_run, repository['path'], storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, progress=compact_arguments.progress, @@ -52,7 +55,9 @@ def run_compact( threshold=compact_arguments.threshold, ) else: # pragma: nocover - logger.info(f'{repository["path"]}: Skipping compact (only available/needed in Borg 1.2+)') + logger.info( + f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)' + ) borgmatic.hooks.command.execute_hook( hooks.get('after_compact'), hooks.get('umask'), diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 3fbe31e..cb8b1cf 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -42,7 +42,7 @@ def run_create( global_arguments.dry_run, **hook_context, ) - logger.info(f'{repository["path"]}: Creating archive{dry_run_label}') + logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}') borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_database_dumps', hooks, @@ -67,6 +67,7 @@ def run_create( location, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, progress=create_arguments.progress, diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index ff9f31b..798bd41 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -33,6 +33,7 @@ def run_export_tar( export_tar_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ), @@ -40,6 +41,7 @@ def run_export_tar( export_tar_arguments.destination, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, tar_filter=export_tar_arguments.tar_filter, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index cc1516c..0bb1efb 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -35,7 +35,9 @@ def run_extract( if extract_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, extract_arguments.repository ): - logger.info(f'{repository["path"]}: Extracting archive {extract_arguments.archive}') + logger.info( + f'{repository.get("label", repository["path"])}: Extracting archive {extract_arguments.archive}' + ) borgmatic.borg.extract.extract_archive( global_arguments.dry_run, repository['path'], @@ -44,6 +46,7 @@ def run_extract( extract_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ), @@ -51,6 +54,7 @@ def run_extract( location, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, destination_path=extract_arguments.destination, diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index 5402312..9169962 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -13,6 +13,7 @@ def run_info( storage, local_borg_version, info_arguments, + global_arguments, local_path, remote_path, ): @@ -25,12 +26,15 @@ def run_info( repository, info_arguments.repository ): if not info_arguments.json: # pragma: nocover - logger.answer(f'{repository["path"]}: Displaying archive summary information') + logger.answer( + f'{repository.get("label", repository["path"])}: Displaying archive summary information' + ) info_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( repository['path'], info_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) @@ -38,9 +42,10 @@ def run_info( repository['path'], storage, local_borg_version, - info_arguments=info_arguments, - local_path=local_path, - remote_path=remote_path, + info_arguments, + global_arguments, + local_path, + remote_path, ) if json_output: # pragma: nocover yield json.loads(json_output) diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index 359f3b6..720fab1 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -12,6 +12,7 @@ def run_list( storage, local_borg_version, list_arguments, + global_arguments, local_path, remote_path, ): @@ -25,14 +26,15 @@ def run_list( ): if not list_arguments.json: # pragma: nocover if list_arguments.find_paths: - logger.answer(f'{repository["path"]}: Searching archives') + logger.answer(f'{repository.get("label", repository["path"])}: Searching archives') elif not list_arguments.archive: - logger.answer(f'{repository["path"]}: Listing archives') + logger.answer(f'{repository.get("label", repository["path"])}: Listing archives') list_arguments.archive = borgmatic.borg.rlist.resolve_archive_name( repository['path'], list_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) @@ -40,9 +42,10 @@ def run_list( repository['path'], storage, local_borg_version, - list_arguments=list_arguments, - local_path=local_path, - remote_path=remote_path, + list_arguments, + global_arguments, + local_path, + remote_path, ) if json_output: # pragma: nocover yield json.loads(json_output) diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index ec472eb..cc8a2cb 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -12,6 +12,7 @@ def run_mount( storage, local_borg_version, mount_arguments, + global_arguments, local_path, remote_path, ): @@ -22,9 +23,11 @@ def run_mount( repository, mount_arguments.repository ): if mount_arguments.archive: - logger.info(f'{repository["path"]}: Mounting archive {mount_arguments.archive}') + logger.info( + f'{repository.get("label", repository["path"])}: Mounting archive {mount_arguments.archive}' + ) else: # pragma: nocover - logger.info(f'{repository["path"]}: Mounting repository') + logger.info(f'{repository.get("label", repository["path"])}: Mounting repository') borgmatic.borg.mount.mount_archive( repository['path'], @@ -33,12 +36,14 @@ def run_mount( mount_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ), mount_arguments, storage, local_borg_version, + global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index 644fd2a..f728084 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -37,13 +37,14 @@ def run_prune( global_arguments.dry_run, **hook_context, ) - logger.info(f'{repository["path"]}: Pruning archives{dry_run_label}') + logger.info(f'{repository.get("label", repository["path"])}: Pruning archives{dry_run_label}') borgmatic.borg.prune.prune_archives( global_arguments.dry_run, repository['path'], storage, retention, local_borg_version, + global_arguments, prune_arguments, local_path=local_path, remote_path=remote_path, diff --git a/borgmatic/actions/rcreate.py b/borgmatic/actions/rcreate.py index 6220631..1bfc489 100644 --- a/borgmatic/actions/rcreate.py +++ b/borgmatic/actions/rcreate.py @@ -23,12 +23,13 @@ def run_rcreate( ): return - logger.info(f'{repository["path"]}: Creating repository') + logger.info(f'{repository.get("label", repository["path"])}: Creating repository') borgmatic.borg.rcreate.create_repository( global_arguments.dry_run, repository['path'], storage, local_borg_version, + global_arguments, rcreate_arguments.encryption_mode, rcreate_arguments.source_repository, rcreate_arguments.copy_crypt_key, diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index f061dca..ded83f4 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -73,12 +73,14 @@ def restore_single_database( Given (among other things) an archive name, a database hook name, and a configured database configuration dict, restore that database from the archive. ''' - logger.info(f'{repository}: Restoring database {database["name"]}') + logger.info( + f'{repository.get("label", repository["path"])}: Restoring database {database["name"]}' + ) dump_pattern = borgmatic.hooks.dispatch.call_hooks( 'make_database_dump_pattern', hooks, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, database['name'], @@ -87,12 +89,13 @@ def restore_single_database( # Kick off a single database extract to stdout. extract_process = borgmatic.borg.extract.extract_archive( dry_run=global_arguments.dry_run, - repository=repository, + repository=repository['path'], archive=archive_name, paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]), location_config=location, storage_config=storage, local_borg_version=local_borg_version, + global_arguments=global_arguments, local_path=local_path, remote_path=remote_path, destination_path='/', @@ -105,7 +108,7 @@ def restore_single_database( borgmatic.hooks.dispatch.call_hooks( 'restore_database_dump', {hook_name: [database]}, - repository, + repository['path'], borgmatic.hooks.dump.DATABASE_HOOK_NAMES, location, global_arguments.dry_run, @@ -119,14 +122,15 @@ def collect_archive_database_names( location, storage, local_borg_version, + global_arguments, local_path, remote_path, ): ''' Given a local or remote repository path, a resolved archive name, a location configuration dict, - a storage configuration dict, the local Borg version, and local and remote Borg paths, query the - archive for the names of databases it contains and return them as a dict from hook name to a - sequence of database names. + a storage configuration dict, the local Borg version, global_arguments an argparse.Namespace, + and local and remote Borg paths, query the archive for the names of databases it contains and + return them as a dict from hook name to a sequence of database names. ''' borgmatic_source_directory = os.path.expanduser( location.get( @@ -141,6 +145,7 @@ def collect_archive_database_names( archive, storage, local_borg_version, + global_arguments, list_path=parent_dump_path, local_path=local_path, remote_path=remote_path, @@ -262,7 +267,7 @@ def run_restore( return logger.info( - f'{repository["path"]}: Restoring databases from archive {restore_arguments.archive}' + f'{repository.get("label", repository["path"])}: Restoring databases from archive {restore_arguments.archive}' ) borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( @@ -279,6 +284,7 @@ def run_restore( restore_arguments.archive, storage, local_borg_version, + global_arguments, local_path, remote_path, ) @@ -288,6 +294,7 @@ def run_restore( location, storage, local_borg_version, + global_arguments, local_path, remote_path, ) @@ -309,7 +316,7 @@ def run_restore( found_names.add(database_name) restore_single_database( - repository['path'], + repository, location, storage, hooks, @@ -338,7 +345,7 @@ def run_restore( database['name'] = database_name restore_single_database( - repository['path'], + repository, location, storage, hooks, diff --git a/borgmatic/actions/rinfo.py b/borgmatic/actions/rinfo.py index 0947ec3..7756efd 100644 --- a/borgmatic/actions/rinfo.py +++ b/borgmatic/actions/rinfo.py @@ -12,6 +12,7 @@ def run_rinfo( storage, local_borg_version, rinfo_arguments, + global_arguments, local_path, remote_path, ): @@ -24,13 +25,16 @@ def run_rinfo( repository, rinfo_arguments.repository ): if not rinfo_arguments.json: # pragma: nocover - logger.answer(f'{repository["path"]}: Displaying repository summary information') + logger.answer( + f'{repository.get("label", repository["path"])}: Displaying repository summary information' + ) json_output = borgmatic.borg.rinfo.display_repository_info( repository['path'], storage, local_borg_version, rinfo_arguments=rinfo_arguments, + global_arguments=global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/rlist.py b/borgmatic/actions/rlist.py index 10d06a5..a9dee21 100644 --- a/borgmatic/actions/rlist.py +++ b/borgmatic/actions/rlist.py @@ -12,6 +12,7 @@ def run_rlist( storage, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ): @@ -24,13 +25,14 @@ def run_rlist( repository, rlist_arguments.repository ): if not rlist_arguments.json: # pragma: nocover - logger.answer(f'{repository["path"]}: Listing repository') + logger.answer(f'{repository.get("label", repository["path"])}: Listing repository') json_output = borgmatic.borg.rlist.list_repository( repository['path'], storage, local_borg_version, rlist_arguments=rlist_arguments, + global_arguments=global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 8089fd4..df481e4 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,13 +17,16 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' - logger.info(f'{repository["path"]}: Transferring archives to repository') + logger.info( + f'{repository.get("label", repository["path"])}: Transferring archives to repository' + ) borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, repository['path'], storage, local_borg_version, transfer_arguments, + global_arguments, local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index f19d655..82fecc1 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -1,8 +1,9 @@ import logging +import borgmatic.commands.arguments import borgmatic.logger from borgmatic.borg import environment, flags -from borgmatic.execute import execute_command +from borgmatic.execute import DO_NOT_CAPTURE, execute_command logger = logging.getLogger(__name__) @@ -36,6 +37,14 @@ def run_arbitrary_borg( command_options_start_index = 2 if options[0] in BORG_SUBCOMMANDS_WITH_SUBCOMMANDS else 1 borg_command = tuple(options[:command_options_start_index]) command_options = tuple(options[command_options_start_index:]) + + if ( + borg_command + and borg_command[0] in borgmatic.commands.arguments.SUBPARSER_ALIASES.keys() + ): + logger.warning( + f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}" + ) except IndexError: borg_command = () command_options = () @@ -62,7 +71,7 @@ def run_arbitrary_borg( return execute_command( full_command, - output_log_level=logging.ANSWER, + output_file=DO_NOT_CAPTURE, borg_local_path=local_path, extra_environment=environment.make_environment(storage_config), ) diff --git a/borgmatic/borg/break_lock.py b/borgmatic/borg/break_lock.py index 7099af8..3c36195 100644 --- a/borgmatic/borg/break_lock.py +++ b/borgmatic/borg/break_lock.py @@ -10,13 +10,14 @@ def break_lock( repository_path, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, a storage configuration dict, the local Borg version, - and optional local and remote Borg paths, break any repository and cache locks leftover from Borg - aborting. + an argparse.Namespace of global arguments, and optional local and remote Borg paths, break any + repository and cache locks leftover from Borg aborting. ''' umask = storage_config.get('umask', None) lock_wait = storage_config.get('lock_wait', None) @@ -25,6 +26,7 @@ def break_lock( (local_path, 'break-lock') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index cee9d92..930c82b 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -1,5 +1,7 @@ import argparse import datetime +import hashlib +import itertools import json import logging import os @@ -88,12 +90,18 @@ def parse_frequency(frequency): def filter_checks_on_frequency( - location_config, consistency_config, borg_repository_id, checks, force + location_config, + consistency_config, + borg_repository_id, + checks, + force, + archives_check_id=None, ): ''' Given a location config, a consistency config with a "checks" sequence of dicts, a Borg - repository ID, a sequence of checks, and whether to force checks to run, filter down those - checks based on the configured "frequency" for each check as compared to its check time file. + repository ID, a sequence of checks, whether to force checks to run, and an ID for the archives + check potentially being run (if any), filter down those checks based on the configured + "frequency" for each check as compared to its check time file. In other words, a check whose check time file's timestamp is too new (based on the configured frequency) will get cut from the returned sequence of checks. Example: @@ -127,8 +135,8 @@ def filter_checks_on_frequency( if not frequency_delta: continue - check_time = read_check_time( - make_check_time_path(location_config, borg_repository_id, check) + check_time = probe_for_check_time( + location_config, borg_repository_id, check, archives_check_id ) if not check_time: continue @@ -145,36 +153,19 @@ def filter_checks_on_frequency( return tuple(filtered_checks) -def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None): +def make_archive_filter_flags( + local_borg_version, storage_config, checks, check_last=None, prefix=None +): ''' Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the check last value, and a consistency check prefix, transform the checks into tuple of - command-line flags. + command-line flags for filtering archives in a check command. - For example, given parsed checks of: - - ('repository',) - - This will be returned as: - - ('--repository-only',) - - However, if both "repository" and "archives" are in checks, then omit them from the returned - flags because Borg does both checks by default. If "data" is in checks, that implies "archives". - - Additionally, if a check_last value is given and "archives" is in checks, then include a - "--last" flag. And if a prefix value is given and "archives" is in checks, then include a - "--match-archives" flag. + If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if + a prefix value is given and "archives" is in checks, then include a "--match-archives" flag. ''' - if 'data' in checks: - data_flags = ('--verify-data',) - checks += ('archives',) - else: - data_flags = () - - if 'archives' in checks: - last_flags = ('--last', str(check_last)) if check_last else () - match_archives_flags = ( + if 'archives' in checks or 'data' in checks: + return (('--last', str(check_last)) if check_last else ()) + ( ( ('--match-archives', f'sh:{prefix}*') if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version) @@ -189,19 +180,53 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None ) ) ) - else: - last_flags = () - match_archives_flags = () - if check_last: - logger.warning( - 'Ignoring check_last option, as "archives" or "data" are not in consistency checks' - ) - if prefix: - logger.warning( - 'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks' - ) - common_flags = last_flags + match_archives_flags + data_flags + if check_last: + logger.warning( + 'Ignoring check_last option, as "archives" or "data" are not in consistency checks' + ) + if prefix: + logger.warning( + 'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks' + ) + + return () + + +def make_archives_check_id(archive_filter_flags): + ''' + Given a sequence of flags to filter archives, return a unique hash corresponding to those + particular flags. If there are no flags, return None. + ''' + if not archive_filter_flags: + return None + + return hashlib.sha256(' '.join(archive_filter_flags).encode()).hexdigest() + + +def make_check_flags(checks, archive_filter_flags): + ''' + Given a parsed sequence of checks and a sequence of flags to filter archives, transform the + checks into tuple of command-line check flags. + + For example, given parsed checks of: + + ('repository',) + + This will be returned as: + + ('--repository-only',) + + However, if both "repository" and "archives" are in checks, then omit them from the returned + flags because Borg does both checks by default. If "data" is in checks, that implies "archives". + ''' + if 'data' in checks: + data_flags = ('--verify-data',) + checks += ('archives',) + else: + data_flags = () + + common_flags = archive_filter_flags + data_flags if {'repository', 'archives'}.issubset(set(checks)): return common_flags @@ -212,18 +237,27 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None ) -def make_check_time_path(location_config, borg_repository_id, check_type): +def make_check_time_path(location_config, borg_repository_id, check_type, archives_check_id=None): ''' - Given a location configuration dict, a Borg repository ID, and the name of a check type - ("repository", "archives", etc.), return a path for recording that check's time (the time of - that check last occurring). + Given a location configuration dict, a Borg repository ID, the name of a check type + ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a + path for recording that check's time (the time of that check last occurring). ''' + borgmatic_source_directory = os.path.expanduser( + location_config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY) + ) + + if check_type in ('archives', 'data'): + return os.path.join( + borgmatic_source_directory, + 'checks', + borg_repository_id, + check_type, + archives_check_id if archives_check_id else 'all', + ) + return os.path.join( - os.path.expanduser( - location_config.get( - 'borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY - ) - ), + borgmatic_source_directory, 'checks', borg_repository_id, check_type, @@ -253,12 +287,81 @@ def read_check_time(path): return None +def probe_for_check_time(location_config, borg_repository_id, check, archives_check_id): + ''' + Given a location configuration dict, a Borg repository ID, the name of a check type + ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a + the corresponding check time or None if such a check time does not exist. + + When the check type is "archives" or "data", this function probes two different paths to find + the check time, e.g.: + + ~/.borgmatic/checks/1234567890/archives/9876543210 + ~/.borgmatic/checks/1234567890/archives/all + + ... and returns the maximum modification time of the files found (if any). The first path + represents a more specific archives check time (a check on a subset of archives), and the second + is a fallback to the last "all" archives check. + + For other check types, this function reads from a single check time path, e.g.: + + ~/.borgmatic/checks/1234567890/repository + ''' + check_times = ( + read_check_time(group[0]) + for group in itertools.groupby( + ( + make_check_time_path(location_config, borg_repository_id, check, archives_check_id), + make_check_time_path(location_config, borg_repository_id, check), + ) + ) + ) + + try: + return max(check_time for check_time in check_times if check_time) + except ValueError: + return None + + +def upgrade_check_times(location_config, borg_repository_id): + ''' + Given a location configuration dict and a Borg repository ID, upgrade any corresponding check + times on disk from old-style paths to new-style paths. + + Currently, the only upgrade performed is renaming an archive or data check path that looks like: + + ~/.borgmatic/checks/1234567890/archives + + to: + + ~/.borgmatic/checks/1234567890/archives/all + ''' + for check_type in ('archives', 'data'): + new_path = make_check_time_path(location_config, borg_repository_id, check_type, 'all') + old_path = os.path.dirname(new_path) + temporary_path = f'{old_path}.temp' + + if not os.path.isfile(old_path) and not os.path.isfile(temporary_path): + return + + logger.debug(f'Upgrading archives check time from {old_path} to {new_path}') + + try: + os.rename(old_path, temporary_path) + except FileNotFoundError: + pass + + os.mkdir(old_path) + os.rename(temporary_path, new_path) + + def check_archives( repository_path, location_config, storage_config, consistency_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, progress=None, @@ -283,6 +386,7 @@ def check_archives( storage_config, local_borg_version, argparse.Namespace(json=True), + global_arguments, local_path, remote_path, ) @@ -290,16 +394,26 @@ def check_archives( except (json.JSONDecodeError, KeyError): raise ValueError(f'Cannot determine Borg repository ID for {repository_path}') + upgrade_check_times(location_config, borg_repository_id) + + check_last = consistency_config.get('check_last', None) + prefix = consistency_config.get('prefix') + configured_checks = parse_checks(consistency_config, only_checks) + lock_wait = None + extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') + archive_filter_flags = make_archive_filter_flags( + local_borg_version, storage_config, configured_checks, check_last, prefix + ) + archives_check_id = make_archives_check_id(archive_filter_flags) + checks = filter_checks_on_frequency( location_config, consistency_config, borg_repository_id, - parse_checks(consistency_config, only_checks), + configured_checks, force, + archives_check_id, ) - check_last = consistency_config.get('check_last', None) - lock_wait = None - extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '') if set(checks).intersection({'repository', 'archives', 'data'}): lock_wait = storage_config.get('lock_wait') @@ -310,13 +424,12 @@ def check_archives( if logger.isEnabledFor(logging.DEBUG): verbosity_flags = ('--debug', '--show-rc') - prefix = consistency_config.get('prefix') - full_command = ( (local_path, 'check') + (('--repair',) if repair else ()) - + make_check_flags(local_borg_version, storage_config, checks, check_last, prefix) + + make_check_flags(checks, archive_filter_flags) + (('--remote-path', remote_path) if remote_path else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags + (('--progress',) if progress else ()) @@ -336,10 +449,18 @@ def check_archives( execute_command(full_command, extra_environment=borg_environment) for check in checks: - write_check_time(make_check_time_path(location_config, borg_repository_id, check)) + write_check_time( + make_check_time_path(location_config, borg_repository_id, check, archives_check_id) + ) if 'extract' in checks: extract.extract_last_archive_dry_run( - storage_config, local_borg_version, repository_path, lock_wait, local_path, remote_path + storage_config, + local_borg_version, + global_arguments, + repository_path, + lock_wait, + local_path, + remote_path, ) write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract')) diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index 0e9d3e8..24f37ee 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -11,6 +11,7 @@ def compact_segments( repository_path, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, progress=False, @@ -29,6 +30,7 @@ def compact_segments( (local_path, 'compact') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--progress',) if progress else ()) + (('--cleanup-commits',) if cleanup_commits else ()) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 7413dfc..e3b70eb 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -326,6 +326,7 @@ def create_archive( location_config, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, progress=False, @@ -438,6 +439,7 @@ def create_archive( + (('--files-cache', files_cache) if files_cache else ()) + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--list', '--filter', list_filter_flags) diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index a624f07..b6d9a04 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -15,6 +15,7 @@ def export_tar_archive( destination_path, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, tar_filter=None, @@ -38,6 +39,7 @@ def export_tar_archive( (local_path, 'export-tar') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--list',) if list_files else ()) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index f947141..d5465bb 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -2,6 +2,7 @@ import logging import os import subprocess +import borgmatic.config.validate from borgmatic.borg import environment, feature, flags, rlist from borgmatic.execute import DO_NOT_CAPTURE, execute_command @@ -11,6 +12,7 @@ logger = logging.getLogger(__name__) def extract_last_archive_dry_run( storage_config, local_borg_version, + global_arguments, repository_path, lock_wait=None, local_path='borg', @@ -20,8 +22,6 @@ def extract_last_archive_dry_run( Perform an extraction dry-run of the most recent archive. If there are no archives, skip the dry-run. ''' - remote_path_flags = ('--remote-path', remote_path) if remote_path else () - lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else () verbosity_flags = () if logger.isEnabledFor(logging.DEBUG): verbosity_flags = ('--debug', '--show-rc') @@ -30,7 +30,13 @@ def extract_last_archive_dry_run( try: last_archive_name = rlist.resolve_archive_name( - repository_path, 'latest', storage_config, local_borg_version, local_path, remote_path + repository_path, + 'latest', + storage_config, + local_borg_version, + global_arguments, + local_path, + remote_path, ) except ValueError: logger.warning('No archives found. Skipping extract consistency check.') @@ -40,8 +46,9 @@ def extract_last_archive_dry_run( borg_environment = environment.make_environment(storage_config) full_extract_command = ( (local_path, 'extract', '--dry-run') - + remote_path_flags - + lock_wait_flags + + (('--remote-path', remote_path) if remote_path else ()) + + (('--log-json',) if global_arguments.log_json else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags + list_flag + flags.make_repository_archive_flags( @@ -62,6 +69,7 @@ def extract_archive( location_config, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, destination_path=None, @@ -71,9 +79,9 @@ def extract_archive( ): ''' Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to - restore from the archive, the local Borg version string, location/storage configuration dicts, - optional local and remote Borg paths, and an optional destination path to extract to, extract - the archive into the current directory. + restore from the archive, the local Borg version string, an argparse.Namespace of global + arguments, location/storage configuration dicts, optional local and remote Borg paths, and an + optional destination path to extract to, extract the archive into the current directory. If extract to stdout is True, then start the extraction streaming to stdout, and return that extract process as an instance of subprocess.Popen. @@ -101,6 +109,7 @@ def extract_archive( + (('--remote-path', remote_path) if remote_path else ()) + numeric_ids_flags + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) @@ -109,7 +118,9 @@ def extract_archive( + (('--progress',) if progress else ()) + (('--stdout',) if extract_to_stdout else ()) + flags.make_repository_archive_flags( - repository, + # Make the repository path absolute so the working directory changes below don't + # prevent Borg from finding the repo. + borgmatic.config.validate.normalize_repository_path(repository), archive, local_borg_version, ) diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index ef2c0c4..91520e0 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -12,13 +12,14 @@ def display_archives_info( storage_config, local_borg_version, info_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, and the - arguments to the info action, display summary information for Borg archives in the repository or - return JSON summary information. + Given a local or remote repository path, a storage config dict, the local Borg version, global + arguments as an argparse.Namespace, and the arguments to the info action, display summary + information for Borg archives in the repository or return JSON summary information. ''' borgmatic.logger.add_custom_log_levels() lock_wait = storage_config.get('lock_wait', None) @@ -36,6 +37,7 @@ def display_archives_info( else () ) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', lock_wait) + ( ( diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 908f8fe..96a6a87 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -25,6 +25,7 @@ def make_list_command( storage_config, local_borg_version, list_arguments, + global_arguments, local_path='borg', remote_path=None, ): @@ -48,6 +49,7 @@ def make_list_command( else () ) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', lock_wait) + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES) + ( @@ -90,14 +92,16 @@ def capture_archive_listing( archive, storage_config, local_borg_version, + global_arguments, list_path=None, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, an archive name, a storage config dict, the local Borg - version, the archive path in which to list files, and local and remote Borg paths, capture the - output of listing that archive and return it as a list of file paths. + version, global arguments as an argparse.Namespace, the archive path in which to list files, and + local and remote Borg paths, capture the output of listing that archive and return it as a list + of file paths. ''' borg_environment = environment.make_environment(storage_config) @@ -115,6 +119,7 @@ def capture_archive_listing( json=None, format='{path}{NL}', # noqa: FS003 ), + global_arguments, local_path, remote_path, ), @@ -130,15 +135,17 @@ def list_archive( storage_config, local_borg_version, list_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, the - arguments to the list action, and local and remote Borg paths, display the output of listing - the files of a Borg archive (or return JSON output). If list_arguments.find_paths are given, - list the files by searching across multiple archives. If neither find_paths nor archive name - are given, instead list the archives in the given repository. + Given a local or remote repository path, a storage config dict, the local Borg version, global + arguments as an argparse.Namespace, the arguments to the list action as an argparse.Namespace, + and local and remote Borg paths, display the output of listing the files of a Borg archive (or + return JSON output). If list_arguments.find_paths are given, list the files by searching across + multiple archives. If neither find_paths nor archive name are given, instead list the archives + in the given repository. ''' borgmatic.logger.add_custom_log_levels() @@ -164,6 +171,7 @@ def list_archive( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ) @@ -205,6 +213,7 @@ def list_archive( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ), @@ -233,6 +242,7 @@ def list_archive( storage_config, local_borg_version, archive_arguments, + global_arguments, local_path, remote_path, ) + make_find_paths(list_arguments.find_paths) diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 3302617..80cfa8d 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -12,14 +12,15 @@ def mount_archive( mount_arguments, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, an optional archive name, a filesystem mount point, zero or more paths to mount from the archive, extra Borg mount options, a storage configuration - dict, the local Borg version, and optional local and remote Borg paths, mount the archive onto - the mount point. + dict, the local Borg version, global arguments as an argparse.Namespace instance, and optional + local and remote Borg paths, mount the archive onto the mount point. ''' umask = storage_config.get('umask', None) lock_wait = storage_config.get('lock_wait', None) @@ -28,6 +29,7 @@ def mount_archive( (local_path, 'mount') + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index c7d1393..a85cacf 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -54,6 +54,7 @@ def prune_archives( retention_config, local_borg_version, prune_arguments, + global_arguments, local_path='borg', remote_path=None, ): @@ -72,6 +73,7 @@ def prune_archives( + make_prune_flags(storage_config, retention_config, local_borg_version) + (('--remote-path', remote_path) if remote_path else ()) + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--stats',) if prune_arguments.stats and not dry_run else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) diff --git a/borgmatic/borg/rcreate.py b/borgmatic/borg/rcreate.py index 7510529..54a865c 100644 --- a/borgmatic/borg/rcreate.py +++ b/borgmatic/borg/rcreate.py @@ -16,6 +16,7 @@ def create_repository( repository_path, storage_config, local_borg_version, + global_arguments, encryption_mode, source_repository=None, copy_crypt_key=False, @@ -37,6 +38,7 @@ def create_repository( storage_config, local_borg_version, argparse.Namespace(json=True), + global_arguments, local_path, remote_path, ) @@ -46,6 +48,7 @@ def create_repository( if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE: raise + lock_wait = storage_config.get('lock_wait') extra_borg_options = storage_config.get('extra_borg_options', {}).get('rcreate', '') rcreate_command = ( @@ -63,6 +66,8 @@ def create_repository( + (('--make-parent-dirs',) if make_parent_dirs else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + + (('--log-json',) if global_arguments.log_json else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--remote-path', remote_path) if remote_path else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) diff --git a/borgmatic/borg/rinfo.py b/borgmatic/borg/rinfo.py index 97d7a66..e1542d2 100644 --- a/borgmatic/borg/rinfo.py +++ b/borgmatic/borg/rinfo.py @@ -12,13 +12,14 @@ def display_repository_info( storage_config, local_borg_version, rinfo_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, a storage config dict, the local Borg version, and the - arguments to the rinfo action, display summary information for the Borg repository or return - JSON summary information. + Given a local or remote repository path, a storage config dict, the local Borg version, the + arguments to the rinfo action, and global arguments as an argparse.Namespace, display summary + information for the Borg repository or return JSON summary information. ''' borgmatic.logger.add_custom_log_levels() lock_wait = storage_config.get('lock_wait', None) @@ -41,6 +42,7 @@ def display_repository_info( else () ) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', lock_wait) + (('--json',) if rinfo_arguments.json else ()) + flags.make_repository_flags(repository_path, local_borg_version) diff --git a/borgmatic/borg/rlist.py b/borgmatic/borg/rlist.py index 7f46870..ba45aa0 100644 --- a/borgmatic/borg/rlist.py +++ b/borgmatic/borg/rlist.py @@ -12,28 +12,29 @@ def resolve_archive_name( archive, storage_config, local_borg_version, + global_arguments, local_path='borg', remote_path=None, ): ''' - Given a local or remote repository path, an archive name, a storage config dict, a local Borg - path, and a remote Borg path, return the archive name. But if the archive name is "latest", - then instead introspect the repository for the latest archive and return its name. + Given a local or remote repository path, an archive name, a storage config dict, the local Borg + version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path, + return the archive name. But if the archive name is "latest", then instead introspect the + repository for the latest archive and return its name. Raise ValueError if "latest" is given but there are no archives in the repository. ''' if archive != 'latest': return archive - lock_wait = storage_config.get('lock_wait', None) - full_command = ( ( local_path, 'rlist' if feature.available(feature.Feature.RLIST, local_borg_version) else 'list', ) + flags.make_flags('remote-path', remote_path) - + flags.make_flags('lock-wait', lock_wait) + + flags.make_flags('log-json', global_arguments.log_json) + + flags.make_flags('lock-wait', storage_config.get('lock_wait')) + flags.make_flags('last', 1) + ('--short',) + flags.make_repository_flags(repository_path, local_borg_version) @@ -61,16 +62,15 @@ def make_rlist_command( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, a storage config dict, the local Borg version, the - arguments to the rlist action, and local and remote Borg paths, return a command as a tuple to - list archives with a repository. + arguments to the rlist action, global arguments as an argparse.Namespace instance, and local and + remote Borg paths, return a command as a tuple to list archives with a repository. ''' - lock_wait = storage_config.get('lock_wait', None) - return ( ( local_path, @@ -87,7 +87,8 @@ def make_rlist_command( else () ) + flags.make_flags('remote-path', remote_path) - + flags.make_flags('lock-wait', lock_wait) + + flags.make_flags('log-json', global_arguments.log_json) + + flags.make_flags('lock-wait', storage_config.get('lock_wait')) + ( ( flags.make_flags('match-archives', f'sh:{rlist_arguments.prefix}*') @@ -113,13 +114,15 @@ def list_repository( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a local or remote repository path, a storage config dict, the local Borg version, the - arguments to the list action, and local and remote Borg paths, display the output of listing - Borg archives in the given repository (or return JSON output). + arguments to the list action, global arguments as an argparse.Namespace instance, and local and + remote Borg paths, display the output of listing Borg archives in the given repository (or + return JSON output). ''' borgmatic.logger.add_custom_log_levels() borg_environment = environment.make_environment(storage_config) @@ -129,6 +132,7 @@ def list_repository( storage_config, local_borg_version, rlist_arguments, + global_arguments, local_path, remote_path, ) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 9fd05b7..d8f3978 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -13,12 +13,14 @@ def transfer_archives( storage_config, local_borg_version, transfer_arguments, + global_arguments, local_path='borg', remote_path=None, ): ''' Given a dry-run flag, a local or remote repository path, a storage config dict, the local Borg - version, and the arguments to the transfer action, transfer archives to the given repository. + version, the arguments to the transfer action, and global arguments as an argparse.Namespace + instance, transfer archives to the given repository. ''' borgmatic.logger.add_custom_log_levels() @@ -27,6 +29,7 @@ def transfer_archives( + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + flags.make_flags('remote-path', remote_path) + + flags.make_flags('log-json', global_arguments.log_json) + flags.make_flags('lock-wait', storage_config.get('lock_wait', None)) + ( flags.make_flags_from_arguments( diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 4b19135..07d1f6d 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -187,6 +187,11 @@ def make_parsers(): type=str, help='Log format string used for log messages written to the log file', ) + global_group.add_argument( + '--log-json', + action='store_true', + help='Write log messages and console output as one JSON object per log line instead of formatted text', + ) global_group.add_argument( '--override', metavar='SECTION.OPTION=VALUE', @@ -207,6 +212,12 @@ def make_parsers(): action='store_true', help='Show bash completion script and exit', ) + global_group.add_argument( + '--fish-completion', + default=False, + action='store_true', + help='Show fish completion script and exit', + ) global_group.add_argument( '--version', dest='version', @@ -986,6 +997,10 @@ def parse_arguments(*unparsed_arguments): raise ValueError( 'With the create action, only one of --list (--files) and --progress flags can be used.' ) + if 'create' in arguments and arguments['create'].list_files and arguments['create'].json: + raise ValueError( + 'With the create action, only one of --list (--files) and --json flags can be used.' + ) if ( ('list' in arguments and 'rinfo' in arguments and arguments['list'].json) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 999e9d8..965f393 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -113,10 +113,14 @@ def run_configuration(config_filename, config, arguments): while not repo_queue.empty(): repository, retry_num = repo_queue.get() - logger.debug(f'{repository["path"]}: Running actions for repository') + logger.debug( + f'{repository.get("label", repository["path"])}: Running actions for repository' + ) timeout = retry_num * retry_wait if timeout: - logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry') + logger.warning( + f'{repository.get("label", repository["path"])}: Sleeping {timeout}s before next retry' + ) time.sleep(timeout) try: yield from run_actions( @@ -139,14 +143,14 @@ def run_configuration(config_filename, config, arguments): ) tuple( # Consume the generator so as to trigger logging. log_error_records( - f'{repository["path"]}: Error running actions for repository', + f'{repository.get("label", repository["path"])}: Error running actions for repository', error, levelno=logging.WARNING, log_command_error_output=True, ) ) logger.warning( - f'{config_filename}: Retrying... attempt {retry_num + 1}/{retries}' + f'{repository.get("label", repository["path"])}: Retrying... attempt {retry_num + 1}/{retries}' ) continue @@ -154,7 +158,8 @@ def run_configuration(config_filename, config, arguments): return yield from log_error_records( - f'{repository["path"]}: Error running actions for repository', error + f'{repository.get("label", repository["path"])}: Error running actions for repository', + error, ) encountered_error = error error_repository = repository['path'] @@ -398,7 +403,8 @@ def run_actions( repository, storage, local_borg_version, - arguments['mount'], + action_arguments, + global_arguments, local_path, remote_path, ) @@ -420,6 +426,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -429,6 +436,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -438,6 +446,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -447,6 +456,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -455,7 +465,8 @@ def run_actions( repository, storage, local_borg_version, - arguments['break-lock'], + action_arguments, + global_arguments, local_path, remote_path, ) @@ -465,6 +476,7 @@ def run_actions( storage, local_borg_version, action_arguments, + global_arguments, local_path, remote_path, ) @@ -715,6 +727,9 @@ def main(): # pragma: no cover if global_arguments.bash_completion: print(borgmatic.commands.completion.bash_completion()) sys.exit(0) + if global_arguments.fish_completion: + print(borgmatic.commands.completion.fish_completion()) + sys.exit(0) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, parse_logs = load_configurations( diff --git a/borgmatic/commands/completion.py b/borgmatic/commands/completion.py index 1fc976b..4b2f17f 100644 --- a/borgmatic/commands/completion.py +++ b/borgmatic/commands/completion.py @@ -1,12 +1,18 @@ +import shlex +from argparse import Action +from textwrap import dedent + from borgmatic.commands import arguments -UPGRADE_MESSAGE = ''' -Your bash completions script is from a different version of borgmatic than is + +def upgrade_message(language: str, upgrade_command: str, completion_file: str): + return f''' +Your {language} completions script is from a different version of borgmatic than is currently installed. Please upgrade your script so your completions match the command-line flags in your installed borgmatic! Try this to upgrade: - sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE" - source $BASH_SOURCE + {upgrade_command} + source {completion_file} ''' @@ -34,7 +40,11 @@ def bash_completion(): ' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"', ' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"', ' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];' - f' then cat << EOF\n{UPGRADE_MESSAGE}\nEOF', + f''' then cat << EOF\n{upgrade_message( + 'bash', + 'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"', + '$BASH_SOURCE', + )}\nEOF''', ' fi', '}', 'complete_borgmatic() {', @@ -55,3 +65,172 @@ def bash_completion(): '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic', ) ) + + +# fish section + + +def has_file_options(action: Action): + ''' + Given an argparse.Action instance, return True if it takes a file argument. + ''' + return action.metavar in ( + 'FILENAME', + 'PATH', + ) or action.dest in ('config_paths',) + + +def has_choice_options(action: Action): + ''' + Given an argparse.Action instance, return True if it takes one of a predefined set of arguments. + ''' + return action.choices is not None + + +def has_unknown_required_param_options(action: Action): + ''' + A catch-all for options that take a required parameter, but we don't know what the parameter is. + This should be used last. These are actions that take something like a glob, a list of numbers, or a string. + + Actions that match this pattern should not show the normal arguments, because those are unlikely to be valid. + ''' + return ( + action.required is True + or action.nargs + in ( + '+', + '*', + ) + or action.metavar in ('PATTERN', 'KEYS', 'N') + or (action.type is not None and action.default is None) + ) + + +def has_exact_options(action: Action): + return ( + has_file_options(action) + or has_choice_options(action) + or has_unknown_required_param_options(action) + ) + + +def exact_options_completion(action: Action): + ''' + Given an argparse.Action instance, return a completion invocation that forces file completions, options completion, + or just that some value follow the action, if the action takes such an argument and was the last action on the + command line prior to the cursor. + + Otherwise, return an empty string. + ''' + + if not has_exact_options(action): + return '' + + args = ' '.join(action.option_strings) + + if has_file_options(action): + return f'''\ncomplete -c borgmatic -Fr -n "__borgmatic_current_arg {args}"''' + + if has_choice_options(action): + return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_current_arg {args}"''' + + if has_unknown_required_param_options(action): + return f'''\ncomplete -c borgmatic -x -n "__borgmatic_current_arg {args}"''' + + raise ValueError( + f'Unexpected action: {action} passes has_exact_options but has no choices produced' + ) + + +def dedent_strip_as_tuple(string: str): + ''' + Dedent a string, then strip it to avoid requiring your first line to have content, then return a tuple of the string. + Makes it easier to write multiline strings for completions when you join them with a tuple. + ''' + return (dedent(string).strip('\n'),) + + +def fish_completion(): + ''' + Return a fish completion script for the borgmatic command. Produce this by introspecting + borgmatic's command-line argument parsers. + ''' + top_level_parser, subparsers = arguments.make_parsers() + + all_subparsers = ' '.join(action for action in subparsers.choices.keys()) + + exact_option_args = tuple( + ' '.join(action.option_strings) + for subparser in subparsers.choices.values() + for action in subparser._actions + if has_exact_options(action) + ) + tuple( + ' '.join(action.option_strings) + for action in top_level_parser._actions + if len(action.option_strings) > 0 + if has_exact_options(action) + ) + + # Avert your eyes. + return '\n'.join( + dedent_strip_as_tuple( + f''' + function __borgmatic_check_version + set -fx this_filename (status current-filename) + fish -c ' + if test -f "$this_filename" + set this_script (cat $this_filename 2> /dev/null) + set installed_script (borgmatic --fish-completion 2> /dev/null) + if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ] + echo "{upgrade_message( + 'fish', + 'borgmatic --fish-completion | sudo tee $this_filename', + '$this_filename', + )}" + end + end + ' & + end + __borgmatic_check_version + + function __borgmatic_current_arg --description 'Check if any of the given arguments are the last on the command line before the cursor' + set -l all_args (commandline -poc) + # premature optimization to avoid iterating all args if there aren't enough + # to have a last arg beyond borgmatic + if [ (count $all_args) -lt 2 ] + return 1 + end + for arg in $argv + if [ "$arg" = "$all_args[-1]" ] + return 0 + end + end + return 1 + end + + set --local subparser_condition "not __fish_seen_subcommand_from {all_subparsers}" + set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}" + ''' + ) + + ('\n# subparser completions',) + + tuple( + f'''complete -c borgmatic -f -n "$subparser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(subparser.description)}''' + for action_name, subparser in subparsers.choices.items() + ) + + ('\n# global flags',) + + tuple( + # -n is checked in order, so put faster / more likely to be true checks first + f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}''' + for action in top_level_parser._actions + # ignore the noargs action, as this is an impossible completion for fish + if len(action.option_strings) > 0 + if 'Deprecated' not in action.help + ) + + ('\n# subparser flags',) + + tuple( + f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}''' + for action_name, subparser in subparsers.choices.items() + for action in subparser._actions + if 'Deprecated' not in action.help + ) + ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 0cf02b2..903c043 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -30,8 +30,8 @@ properties: items: type: string description: | - List of source directories to backup. Globs and tildes are - expanded. Do not backslash spaces in path names. + List of source directories and files to backup. Globs and + tildes are expanded. Do not backslash spaces in path names. example: - /home - /etc diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 537f4be..b39199f 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -8,6 +8,7 @@ try: except ModuleNotFoundError: # pragma: nocover import importlib.metadata as importlib_metadata +import borgmatic.config from borgmatic.config import environment, load, normalize, override @@ -25,7 +26,9 @@ def schema_filename(): if path.match('config/schema.yaml') ) except StopIteration: - raise FileNotFoundError('Configuration file schema could not be found') + # If the schema wasn't found in the package's files, this is probably a pip editable + # install, so try a different approach to get the schema. + return os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml') def format_json_error_path_element(path_element): diff --git a/docs/how-to/deal-with-very-large-backups.md b/docs/how-to/deal-with-very-large-backups.md index e5962c1..5beb9f2 100644 --- a/docs/how-to/deal-with-very-large-backups.md +++ b/docs/how-to/deal-with-very-large-backups.md @@ -95,6 +95,7 @@ See [Borg's check documentation](https://borgbackup.readthedocs.io/en/stable/usage/check.html) for more information. + ### Check frequency New in version 1.6.2 You can diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index f2ddf01..b565d66 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -81,6 +81,9 @@ If `archive_name_format` is unspecified, the default is `{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a timestamp in a particular format. + +### Archive filtering + New in version 1.7.11 borgmatic uses the `archive_name_format` option to automatically limit which archives get used for actions operating on multiple archives. This prevents, for diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index eb7a620..517f9c7 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -73,7 +73,7 @@ from borgmatic for a configured interval. ### Consistency checks -While not strictly part of monitoring, if you really want confidence that your +While not strictly part of monitoring, if you want confidence that your backups are not only running but are restorable as well, you can configure particular [consistency checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration) diff --git a/docs/how-to/run-arbitrary-borg-commands.md b/docs/how-to/run-arbitrary-borg-commands.md index 0777ebb..ea265ea 100644 --- a/docs/how-to/run-arbitrary-borg-commands.md +++ b/docs/how-to/run-arbitrary-borg-commands.md @@ -89,8 +89,10 @@ borgmatic's `borg` action is not without limitations: * Unlike normal borgmatic actions that support JSON, the `borg` action will not disable certain borgmatic logs to avoid interfering with JSON output. * Unlike other borgmatic actions, the `borg` action captures (and logs) all - output, so interactive prompts or flags like `--progress` will not work as - expected. + output, so interactive prompts and flags like `--progress` will not work as + expected. New in version + 1.7.13 borgmatic now runs the `borg` action without capturing output, + so interactive prompts work. In general, this `borgmatic borg` feature should be considered an escape valve—a feature of second resort. In the long run, it's preferable to wrap diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index be229cb..515f017 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -88,6 +88,7 @@ installing borgmatic: * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic) * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic) * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/) + * [Gentoo](https://packages.gentoo.org/packages/app-backup/borgmatic) * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/) * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic) * [OpenBSD](https://openports.pl/path/sysutils/borgmatic) @@ -334,10 +335,13 @@ Access](https://projects.torsion.org/borgmatic-collective/borgmatic/issues/293). ### Shell completion -borgmatic includes a shell completion script (currently only for Bash) to +borgmatic includes a shell completion script (currently only for Bash and Fish) to support tab-completing borgmatic command-line actions and flags. Depending on -how you installed borgmatic, this may be enabled by default. But if it's not, -start by installing the `bash-completion` Linux package or the +how you installed borgmatic, this may be enabled by default. + +#### Bash + +If completions aren't enabled, start by installing the `bash-completion` Linux package or the [`bash-completion@2`](https://formulae.brew.sh/formula/bash-completion@2) macOS Homebrew formula. Then, install the shell completion script globally: @@ -362,6 +366,14 @@ borgmatic --bash-completion > ~/.local/share/bash-completion/completions/borgmat Finally, restart your shell (`exit` and open a new shell) so the completions take effect. +#### fish + +To add completions for fish, install the completions file globally: + +```fish +borgmatic --fish-completion | sudo tee /usr/share/fish/vendor_completions.d/borgmatic.fish +source /usr/share/fish/vendor_completions.d/borgmatic.fish +``` ### Colored output diff --git a/scripts/run-full-tests b/scripts/run-full-tests index bf26c21..a7a49a2 100755 --- a/scripts/run-full-tests +++ b/scripts/run-full-tests @@ -10,7 +10,7 @@ set -e -if [ -z "$TEST_CONTAINER" ] ; then +if [ -z "$TEST_CONTAINER" ]; then echo "This script is designed to work inside a test container and is not intended to" echo "be run manually. If you're trying to run borgmatic's end-to-end tests, execute" echo "scripts/run-end-to-end-dev-tests instead." @@ -18,14 +18,14 @@ if [ -z "$TEST_CONTAINER" ] ; then fi apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \ - py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite + py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite fish # If certain dependencies of black are available in this version of Alpine, install them. apk add --no-cache py3-typed-ast py3-regex || true python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1 pip3 install --ignore-installed tox==3.25.1 export COVERAGE_FILE=/tmp/.coverage -if [ "$1" != "--end-to-end-only" ] ; then +if [ "$1" != "--end-to-end-only" ]; then tox --workdir /tmp/.tox --sitepackages fi diff --git a/setup.py b/setup.py index ce6b78e..2665af8 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import find_packages, setup -VERSION = '1.7.13.dev0' +VERSION = '1.7.14.dev0' setup( diff --git a/tests/end-to-end/test_completion.py b/tests/end-to-end/test_completion.py index e4037ec..7d6af4c 100644 --- a/tests/end-to-end/test_completion.py +++ b/tests/end-to-end/test_completion.py @@ -3,3 +3,7 @@ import subprocess def test_bash_completion_runs_without_error(): subprocess.check_call('borgmatic --bash-completion | bash', shell=True) + + +def test_fish_completion_runs_without_error(): + subprocess.check_call('borgmatic --fish-completion | fish', shell=True) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 6c330d6..3347890 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -1,3 +1,4 @@ +import argparse import copy from flexmock import flexmock @@ -58,7 +59,12 @@ def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): continue borgmatic.borg.transfer.transfer_archives( - False, 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + False, + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + global_arguments=flexmock(log_json=False), ) @@ -70,7 +76,11 @@ def test_make_list_command_does_not_duplicate_flags_or_raise(): continue command = borgmatic.borg.list.make_list_command( - 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + argparse.Namespace(log_json=False), ) assert_command_does_not_duplicate_flags(command) @@ -84,7 +94,11 @@ def test_make_rlist_command_does_not_duplicate_flags_or_raise(): continue command = borgmatic.borg.rlist.make_rlist_command( - 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + global_arguments=flexmock(log_json=True), ) assert_command_does_not_duplicate_flags(command) @@ -104,7 +118,11 @@ def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): continue borgmatic.borg.info.display_archives_info( - 'repo', {}, '2.3.4', fuzz_argument(arguments, argument_name) + 'repo', + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + argparse.Namespace(log_json=False), ) @@ -119,5 +137,11 @@ def test_prune_archives_command_does_not_duplicate_flags_or_raise(): continue borgmatic.borg.prune.prune_archives( - False, 'repo', {}, {}, '2.3.4', fuzz_argument(arguments, argument_name) + False, + 'repo', + {}, + {}, + '2.3.4', + fuzz_argument(arguments, argument_name), + argparse.Namespace(log_json=False), ) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 1fc8f8c..4990fc4 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -422,6 +422,13 @@ def test_parse_arguments_disallows_list_with_progress_for_create_action(): module.parse_arguments('create', '--list', '--progress') +def test_parse_arguments_disallows_list_with_json_for_create_action(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + with pytest.raises(ValueError): + module.parse_arguments('create', '--list', '--json') + + def test_parse_arguments_allows_json_with_list_or_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff --git a/tests/integration/commands/test_completion.py b/tests/integration/commands/test_completion.py index a3b0b9c..9a118ab 100644 --- a/tests/integration/commands/test_completion.py +++ b/tests/integration/commands/test_completion.py @@ -3,3 +3,7 @@ from borgmatic.commands import completion as module def test_bash_completion_does_not_raise(): assert module.bash_completion() + + +def test_fish_completion_does_not_raise(): + assert module.fish_completion() diff --git a/tests/unit/actions/test_borg.py b/tests/unit/actions/test_borg.py index f597acb..2e03ec9 100644 --- a/tests/unit/actions/test_borg.py +++ b/tests/unit/actions/test_borg.py @@ -16,6 +16,7 @@ def test_run_borg_does_not_raise(): repository={'path': 'repos'}, storage={}, local_borg_version=None, + global_arguments=flexmock(log_json=False), borg_arguments=borg_arguments, local_path=None, remote_path=None, diff --git a/tests/unit/actions/test_break_lock.py b/tests/unit/actions/test_break_lock.py index 6dc2470..5949d7c 100644 --- a/tests/unit/actions/test_break_lock.py +++ b/tests/unit/actions/test_break_lock.py @@ -14,6 +14,7 @@ def test_run_break_lock_does_not_raise(): storage={}, local_borg_version=None, break_lock_arguments=break_lock_arguments, + global_arguments=flexmock(), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_info.py b/tests/unit/actions/test_info.py index a4f1d54..9716196 100644 --- a/tests/unit/actions/test_info.py +++ b/tests/unit/actions/test_info.py @@ -18,6 +18,7 @@ def test_run_info_does_not_raise(): storage={}, local_borg_version=None, info_arguments=info_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_list.py b/tests/unit/actions/test_list.py index bfdfd01..5ee7225 100644 --- a/tests/unit/actions/test_list.py +++ b/tests/unit/actions/test_list.py @@ -18,6 +18,7 @@ def test_run_list_does_not_raise(): storage={}, local_borg_version=None, list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_mount.py b/tests/unit/actions/test_mount.py index 7eadfca..743747d 100644 --- a/tests/unit/actions/test_mount.py +++ b/tests/unit/actions/test_mount.py @@ -21,6 +21,7 @@ def test_run_mount_does_not_raise(): storage={}, local_borg_version=None, mount_arguments=mount_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 16fe292..4bad6f8 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -72,6 +72,7 @@ def test_collect_archive_database_names_parses_archive_paths(): location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), local_path=flexmock(), remote_path=flexmock(), ) @@ -97,6 +98,7 @@ def test_collect_archive_database_names_parses_directory_format_archive_paths(): location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), local_path=flexmock(), remote_path=flexmock(), ) @@ -118,6 +120,7 @@ def test_collect_archive_database_names_skips_bad_archive_paths(): location={'borgmatic_source_directory': '.borgmatic'}, storage=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), local_path=flexmock(), remote_path=flexmock(), ) diff --git a/tests/unit/actions/test_rinfo.py b/tests/unit/actions/test_rinfo.py index 133e61a..7b2371a 100644 --- a/tests/unit/actions/test_rinfo.py +++ b/tests/unit/actions/test_rinfo.py @@ -15,6 +15,7 @@ def test_run_rinfo_does_not_raise(): storage={}, local_borg_version=None, rinfo_arguments=rinfo_arguments, + global_arguments=flexmock(log_json=False), local_path=None, remote_path=None, ) diff --git a/tests/unit/actions/test_rlist.py b/tests/unit/actions/test_rlist.py index 7f8b58a..4a59dc3 100644 --- a/tests/unit/actions/test_rlist.py +++ b/tests/unit/actions/test_rlist.py @@ -15,6 +15,7 @@ def test_run_rlist_does_not_raise(): storage={}, local_borg_version=None, rlist_arguments=rlist_arguments, + global_arguments=flexmock(), local_path=None, remote_path=None, ) diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 5b73596..5ae013f 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -7,7 +7,7 @@ from borgmatic.borg import borg as module from ..test_verbosity import insert_logging_mock -def test_run_arbitrary_borg_calls_borg_with_parameters(): +def test_run_arbitrary_borg_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -15,7 +15,7 @@ def test_run_arbitrary_borg_calls_borg_with_parameters(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -28,7 +28,7 @@ def test_run_arbitrary_borg_calls_borg_with_parameters(): ) -def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): +def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -36,7 +36,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--info'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -50,7 +50,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_parameter(): ) -def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): +def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -58,7 +58,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--debug', '--show-rc'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -72,7 +72,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_parameter(): ) -def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} @@ -83,7 +83,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters( flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--lock-wait', '5'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -96,7 +96,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_parameters( ) -def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter(): +def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -106,7 +106,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_parameter(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo::archive'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -128,7 +128,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg1', 'break-lock', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg1', extra_environment=None, ) @@ -142,7 +142,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): ) -def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -152,7 +152,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo', '--remote-path', 'borg1'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -166,7 +166,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_paramet ) -def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): +def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -174,7 +174,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo', '--progress'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -187,7 +187,7 @@ def test_run_arbitrary_borg_passes_borg_specific_parameters_to_borg(): ) -def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): +def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -195,7 +195,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'break-lock', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -208,7 +208,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_parameters_passed_to_borg(): ) -def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise(): +def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_flags').never() @@ -216,7 +216,7 @@ def test_run_arbitrary_borg_without_borg_specific_parameters_does_not_raise(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg',), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -237,7 +237,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_repository(): flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'key', 'export', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -258,7 +258,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_repository() flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'debug', 'dump-manifest', 'repo', 'path'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -279,7 +279,7 @@ def test_run_arbitrary_borg_with_debug_info_command_does_not_pass_borg_repositor flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'debug', 'info'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) @@ -300,7 +300,7 @@ def test_run_arbitrary_borg_with_debug_convert_profile_command_does_not_pass_bor flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( ('borg', 'debug', 'convert-profile', 'in', 'out'), - output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.borgmatic.execute.DO_NOT_CAPTURE, borg_local_path='borg', extra_environment=None, ) diff --git a/tests/unit/borg/test_break_lock.py b/tests/unit/borg/test_break_lock.py index 509fc1b..3dc5567 100644 --- a/tests/unit/borg/test_break_lock.py +++ b/tests/unit/borg/test_break_lock.py @@ -24,6 +24,7 @@ def test_break_lock_calls_borg_with_required_flags(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -35,6 +36,7 @@ def test_break_lock_calls_borg_with_remote_path_flags(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -47,6 +49,19 @@ def test_break_lock_calls_borg_with_umask_flags(): repository_path='repo', storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_break_lock_calls_borg_with_log_json_flags(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(('borg', 'break-lock', '--log-json', 'repo')) + + module.break_lock( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -58,6 +73,7 @@ def test_break_lock_calls_borg_with_lock_wait_flags(): repository_path='repo', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -70,6 +86,7 @@ def test_break_lock_with_log_info_calls_borg_with_info_parameter(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -82,4 +99,5 @@ def test_break_lock_with_log_debug_calls_borg_with_debug_flags(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 1f992d3..89db5d2 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -96,7 +96,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks(): module.datetime.timedelta(weeks=4) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return(None) + flexmock(module).should_receive('probe_for_check_time').and_return(None) assert module.filter_checks_on_frequency( location_config={}, @@ -104,6 +104,7 @@ def test_filter_checks_on_frequency_without_config_uses_default_checks(): borg_repository_id='repo', checks=('repository', 'archives'), force=False, + archives_check_id='1234', ) == ('repository', 'archives') @@ -126,6 +127,7 @@ def test_filter_checks_on_frequency_retains_check_without_frequency(): borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == ('archives',) @@ -134,7 +136,7 @@ def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency(): module.datetime.timedelta(hours=1) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return( + flexmock(module).should_receive('probe_for_check_time').and_return( module.datetime.datetime(year=module.datetime.MINYEAR, month=1, day=1) ) @@ -144,6 +146,7 @@ def test_filter_checks_on_frequency_retains_check_with_elapsed_frequency(): borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == ('archives',) @@ -152,7 +155,7 @@ def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file() module.datetime.timedelta(hours=1) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return(None) + flexmock(module).should_receive('probe_for_check_time').and_return(None) assert module.filter_checks_on_frequency( location_config={}, @@ -160,6 +163,7 @@ def test_filter_checks_on_frequency_retains_check_with_missing_check_time_file() borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == ('archives',) @@ -168,7 +172,9 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency(): module.datetime.timedelta(hours=1) ) flexmock(module).should_receive('make_check_time_path') - flexmock(module).should_receive('read_check_time').and_return(module.datetime.datetime.now()) + flexmock(module).should_receive('probe_for_check_time').and_return( + module.datetime.datetime.now() + ) assert ( module.filter_checks_on_frequency( @@ -177,6 +183,7 @@ def test_filter_checks_on_frequency_skips_check_with_unelapsed_frequency(): borg_repository_id='repo', checks=('archives',), force=False, + archives_check_id='1234', ) == () ) @@ -189,32 +196,177 @@ def test_filter_checks_on_frequency_restains_check_with_unelapsed_frequency_and_ borg_repository_id='repo', checks=('archives',), force=True, + archives_check_id='1234', ) == ('archives',) -def test_make_check_flags_with_repository_check_returns_flag(): +def test_make_archive_filter_flags_with_default_checks_and_prefix_returns_default_flags(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('repository',)) + flags = module.make_archive_filter_flags( + '1.2.3', + {}, + ('repository', 'archives'), + prefix='foo', + ) + + assert flags == ('--match-archives', 'sh:foo*') + + +def test_make_archive_filter_flags_with_all_checks_and_prefix_returns_default_flags(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags( + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', + ) + + assert flags == ('--match-archives', 'sh:foo*') + + +def test_make_archive_filter_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags(): + flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags( + '1.2.3', + {}, + ('repository', 'archives', 'extract'), + prefix='foo', + ) + + assert flags == ('--glob-archives', 'foo*') + + +def test_make_archive_filter_flags_with_archives_check_and_last_includes_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), check_last=3) + + assert flags == ('--last', '3') + + +def test_make_archive_filter_flags_with_data_check_and_last_includes_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), check_last=3) + + assert flags == ('--last', '3') + + +def test_make_archive_filter_flags_with_repository_check_and_last_omits_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), check_last=3) + + assert flags == () + + +def test_make_archive_filter_flags_with_default_checks_and_last_includes_last_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), check_last=3) + + assert flags == ('--last', '3') + + +def test_make_archive_filter_flags_with_archives_check_and_prefix_includes_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix='foo-') + + assert flags == ('--match-archives', 'sh:foo-*') + + +def test_make_archive_filter_flags_with_data_check_and_prefix_includes_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('data',), prefix='foo-') + + assert flags == ('--match-archives', 'sh:foo-*') + + +def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, 'bar-{now}', '1.2.3' # noqa: FS003 + ).and_return(('--match-archives', 'sh:bar-*')) + + flags = module.make_archive_filter_flags( + '1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003 + ) + + assert flags == ('--match-archives', 'sh:bar-*') + + +def test_make_archive_filter_flags_with_archives_check_and_none_prefix_omits_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('archives',), prefix=None) + + assert flags == () + + +def test_make_archive_filter_flags_with_repository_check_and_prefix_omits_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository',), prefix='foo-') + + assert flags == () + + +def test_make_archive_filter_flags_with_default_checks_and_prefix_includes_match_archives_flag(): + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + + flags = module.make_archive_filter_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-') + + assert flags == ('--match-archives', 'sh:foo-*') + + +def test_make_archives_check_id_with_flags_returns_a_value_and_does_not_raise(): + assert module.make_archives_check_id(('--match-archives', 'sh:foo-*')) + + +def test_make_archives_check_id_with_empty_flags_returns_none(): + assert module.make_archives_check_id(()) is None + + +def test_make_check_flags_with_repository_check_returns_flag(): + flags = module.make_check_flags(('repository',), ()) assert flags == ('--repository-only',) def test_make_check_flags_with_archives_check_returns_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('archives',)) + flags = module.make_check_flags(('archives',), ()) assert flags == ('--archives-only',) +def test_make_check_flags_with_archive_filtler_flags_includes_those_flags(): + flags = module.make_check_flags(('archives',), ('--match-archives', 'sh:foo-*')) + + assert flags == ('--archives-only', '--match-archives', 'sh:foo-*') + + def test_make_check_flags_with_data_check_returns_flag_and_implies_archives(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('data',)) + flags = module.make_check_flags(('data',), ()) assert flags == ( '--archives-only', @@ -226,7 +378,7 @@ def test_make_check_flags_with_extract_omits_extract_flag(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flags = module.make_check_flags('1.2.3', {}, ('extract',)) + flags = module.make_check_flags(('extract',), ()) assert flags == () @@ -236,151 +388,66 @@ def test_make_check_flags_with_repository_and_data_checks_does_not_return_reposi flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flags = module.make_check_flags( - '1.2.3', - {}, ( 'repository', 'data', ), + (), ) assert flags == ('--verify-data',) -def test_make_check_flags_with_default_checks_and_prefix_returns_default_flags(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags( - '1.2.3', - {}, - ('repository', 'archives'), - prefix='foo', +def test_make_check_time_path_with_borgmatic_source_directory_includes_it(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return( + '/home/user/.borgmatic' ) - assert flags == ('--match-archives', 'sh:foo*') - - -def test_make_check_flags_with_all_checks_and_prefix_returns_default_flags(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags( - '1.2.3', - {}, - ('repository', 'archives', 'extract'), - prefix='foo', + assert ( + module.make_check_time_path( + {'borgmatic_source_directory': '~/.borgmatic'}, '1234', 'archives', '5678' + ) + == '/home/user/.borgmatic/checks/1234/archives/5678' ) - assert flags == ('--match-archives', 'sh:foo*') +def test_make_check_time_path_without_borgmatic_source_directory_uses_default(): + flexmock(module.os.path).should_receive('expanduser').with_args( + module.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY + ).and_return('/home/user/.borgmatic') -def test_make_check_flags_with_all_checks_and_prefix_without_borg_features_returns_glob_archives_flags(): - flexmock(module.feature).should_receive('available').and_return(False) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags( - '1.2.3', - {}, - ('repository', 'archives', 'extract'), - prefix='foo', + assert ( + module.make_check_time_path({}, '1234', 'archives', '5678') + == '/home/user/.borgmatic/checks/1234/archives/5678' ) - assert flags == ('--glob-archives', 'foo*') - -def test_make_check_flags_with_archives_check_and_last_includes_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('archives',), check_last=3) - - assert flags == ('--archives-only', '--last', '3') - - -def test_make_check_flags_with_data_check_and_last_includes_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('data',), check_last=3) - - assert flags == ('--archives-only', '--last', '3', '--verify-data') - - -def test_make_check_flags_with_repository_check_and_last_omits_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository',), check_last=3) - - assert flags == ('--repository-only',) - - -def test_make_check_flags_with_default_checks_and_last_includes_last_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), check_last=3) - - assert flags == ('--last', '3') - - -def test_make_check_flags_with_archives_check_and_prefix_includes_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix='foo-') - - assert flags == ('--archives-only', '--match-archives', 'sh:foo-*') - - -def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('data',), prefix='foo-') - - assert flags == ('--archives-only', '--match-archives', 'sh:foo-*', '--verify-data') - - -def test_make_check_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - None, 'bar-{now}', '1.2.3' # noqa: FS003 - ).and_return(('--match-archives', 'sh:bar-*')) - - flags = module.make_check_flags( - '1.2.3', {'archive_name_format': 'bar-{now}'}, ('archives',), prefix='' # noqa: FS003 +def test_make_check_time_path_with_archives_check_and_no_archives_check_id_defaults_to_all(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return( + '/home/user/.borgmatic' ) - assert flags == ('--archives-only', '--match-archives', 'sh:bar-*') + assert ( + module.make_check_time_path( + {'borgmatic_source_directory': '~/.borgmatic'}, + '1234', + 'archives', + ) + == '/home/user/.borgmatic/checks/1234/archives/all' + ) -def test_make_check_flags_with_archives_check_and_none_prefix_omits_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) +def test_make_check_time_path_with_repositories_check_ignores_archives_check_id(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/.borgmatic').and_return( + '/home/user/.borgmatic' + ) - flags = module.make_check_flags('1.2.3', {}, ('archives',), prefix=None) - - assert flags == ('--archives-only',) - - -def test_make_check_flags_with_repository_check_and_prefix_omits_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository',), prefix='foo-') - - assert flags == ('--repository-only',) - - -def test_make_check_flags_with_default_checks_and_prefix_includes_match_archives_flag(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - - flags = module.make_check_flags('1.2.3', {}, ('repository', 'archives'), prefix='foo-') - - assert flags == ('--match-archives', 'sh:foo-*') + assert ( + module.make_check_time_path( + {'borgmatic_source_directory': '~/.borgmatic'}, '1234', 'repository', '5678' + ) + == '/home/user/.borgmatic/checks/1234/repository' + ) def test_read_check_time_does_not_raise(): @@ -395,14 +462,135 @@ def test_read_check_time_on_missing_file_does_not_raise(): assert module.read_check_time('/path') is None +def test_probe_for_check_time_uses_maximum_of_multiple_check_times(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(1).and_return(2) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2 + + +def test_probe_for_check_time_deduplicates_identical_check_time_paths(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/5678') + flexmock(module).should_receive('read_check_time').and_return(1).once() + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1 + + +def test_probe_for_check_time_skips_none_check_time(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(None).and_return(2) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 2 + + +def test_probe_for_check_time_uses_single_check_time(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(1).and_return(None) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) == 1 + + +def test_probe_for_check_time_returns_none_when_no_check_time_found(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/5678' + ).and_return('~/.borgmatic/checks/1234/archives/all') + flexmock(module).should_receive('read_check_time').and_return(None).and_return(None) + + assert module.probe_for_check_time(flexmock(), flexmock(), flexmock(), flexmock()) is None + + +def test_upgrade_check_times_renames_old_check_paths_to_all(): + base_path = '~/.borgmatic/checks/1234' + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'archives', 'all' + ).and_return(f'{base_path}/archives/all') + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'data', 'all' + ).and_return(f'{base_path}/data/all') + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return( + True + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/archives.temp' + ).and_return(False) + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return( + False + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/data.temp' + ).and_return(False) + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives', f'{base_path}/archives.temp' + ).once() + flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once() + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives.temp', f'{base_path}/archives/all' + ).once() + + module.upgrade_check_times(flexmock(), flexmock()) + + +def test_upgrade_check_times_skips_missing_check_paths(): + flexmock(module).should_receive('make_check_time_path').and_return( + '~/.borgmatic/checks/1234/archives/all' + ) + flexmock(module.os.path).should_receive('isfile').and_return(False) + flexmock(module.os).should_receive('rename').never() + flexmock(module.os).should_receive('mkdir').never() + + module.upgrade_check_times(flexmock(), flexmock()) + + +def test_upgrade_check_times_renames_stale_temporary_check_path(): + base_path = '~/.borgmatic/checks/1234' + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'archives', 'all' + ).and_return(f'{base_path}/archives/all') + flexmock(module).should_receive('make_check_time_path').with_args( + object, object, 'data', 'all' + ).and_return(f'{base_path}/data/all') + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/archives').and_return( + False + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/archives.temp' + ).and_return(True) + flexmock(module.os.path).should_receive('isfile').with_args(f'{base_path}/data').and_return( + False + ) + flexmock(module.os.path).should_receive('isfile').with_args( + f'{base_path}/data.temp' + ).and_return(False) + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives', f'{base_path}/archives.temp' + ).and_raise(FileNotFoundError) + flexmock(module.os).should_receive('mkdir').with_args(f'{base_path}/archives').once() + flexmock(module.os).should_receive('rename').with_args( + f'{base_path}/archives.temp', f'{base_path}/archives/all' + ).once() + + module.upgrade_check_times(flexmock(), flexmock()) + + def test_check_archives_with_progress_calls_borg_with_progress_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module).should_receive('execute_command').never() flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -421,6 +609,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -428,11 +617,14 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter(): def test_check_archives_with_repair_calls_borg_with_repair_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module).should_receive('execute_command').never() flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -451,6 +643,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), repair=True, ) @@ -467,18 +660,15 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter(): def test_check_archives_calls_borg_with_parameters(checks): check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - {}, - checks, - check_last, - prefix=None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -490,6 +680,7 @@ def test_check_archives_calls_borg_with_parameters(checks): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -497,11 +688,14 @@ def test_check_archives_with_json_error_raises(): checks = ('archives',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"unexpected": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) with pytest.raises(ValueError): module.check_archives( @@ -510,6 +704,7 @@ def test_check_archives_with_json_error_raises(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -517,9 +712,12 @@ def test_check_archives_with_missing_json_keys_raises(): checks = ('archives',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return('{invalid JSON') + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) with pytest.raises(ValueError): module.check_archives( @@ -528,6 +726,7 @@ def test_check_archives_with_missing_json_keys_raises(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -535,11 +734,14 @@ def test_check_archives_with_extract_check_calls_extract_only(): checks = ('extract',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').never() flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.extract).should_receive('extract_last_archive_dry_run').once() @@ -552,17 +754,21 @@ def test_check_archives_with_extract_check_calls_extract_only(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) def test_check_archives_with_log_info_calls_borg_with_info_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_logging_mock(logging.INFO) @@ -576,17 +782,21 @@ def test_check_archives_with_log_info_calls_borg_with_info_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_logging_mock(logging.DEBUG) @@ -600,16 +810,20 @@ def test_check_archives_with_log_debug_calls_borg_with_debug_parameter(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) def test_check_archives_without_any_checks_bails(): consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(()) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(()) insert_execute_command_never() module.check_archives( @@ -618,6 +832,7 @@ def test_check_archives_without_any_checks_bails(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -625,18 +840,15 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): checks = ('repository',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - {}, - checks, - check_last, - prefix=None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg1', 'check', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -648,6 +860,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -656,18 +869,15 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( checks = ('repository',) check_last = flexmock() consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - {}, - checks, - check_last, - prefix=None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--remote-path', 'borg1', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -679,27 +889,54 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters( storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) +def test_check_archives_with_log_json_calls_borg_with_log_json_parameters(): + checks = ('repository',) + check_last = flexmock() + storage_config = {} + consistency_config = {'check_last': check_last} + flexmock(module.rinfo).should_receive('display_repository_info').and_return( + '{"repository": {"id": "repo"}}' + ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(('borg', 'check', '--log-json', 'repo')) + flexmock(module).should_receive('make_check_time_path') + flexmock(module).should_receive('write_check_time') + + module.check_archives( + repository_path='repo', + location_config={}, + storage_config=storage_config, + consistency_config=consistency_config, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + ) + + def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): checks = ('repository',) check_last = flexmock() storage_config = {'lock_wait': 5} consistency_config = {'check_last': check_last} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', - storage_config, - checks, - check_last, - None, - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--lock-wait', '5', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -711,6 +948,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): storage_config=storage_config, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -719,14 +957,15 @@ def test_check_archives_with_retention_prefix(): check_last = flexmock() prefix = 'foo-' consistency_config = {'check_last': check_last, 'prefix': prefix} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) - flexmock(module).should_receive('make_check_flags').with_args( - '1.2.3', {}, checks, check_last, prefix - ).and_return(()) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) + flexmock(module).should_receive('make_check_flags').with_args(checks, ()).and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', 'repo')) flexmock(module).should_receive('make_check_time_path') @@ -738,17 +977,21 @@ def test_check_archives_with_retention_prefix(): storage_config={}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): checks = ('repository',) consistency_config = {'check_last': None} - flexmock(module).should_receive('parse_checks') - flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module.rinfo).should_receive('display_repository_info').and_return( '{"repository": {"id": "repo"}}' ) + flexmock(module).should_receive('upgrade_check_times') + flexmock(module).should_receive('parse_checks') + flexmock(module).should_receive('make_archive_filter_flags').and_return(()) + flexmock(module).should_receive('make_archives_check_id').and_return(None) + flexmock(module).should_receive('filter_checks_on_frequency').and_return(checks) flexmock(module).should_receive('make_check_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(('borg', 'check', '--extra', '--options', 'repo')) @@ -761,4 +1004,5 @@ def test_check_archives_with_extra_borg_options_calls_borg_with_extra_options(): storage_config={'extra_borg_options': {'check': '--extra --options'}}, consistency_config=consistency_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index 60447db..beacf54 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -25,7 +25,11 @@ def test_compact_segments_calls_borg_with_parameters(): insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.INFO) module.compact_segments( - dry_run=False, repository_path='repo', storage_config={}, local_borg_version='1.2.3' + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -35,7 +39,11 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): insert_logging_mock(logging.INFO) module.compact_segments( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + dry_run=False, ) @@ -45,7 +53,11 @@ def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): insert_logging_mock(logging.DEBUG) module.compact_segments( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=False + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + dry_run=False, ) @@ -53,7 +65,11 @@ def test_compact_segments_with_dry_run_skips_borg_call(): flexmock(module).should_receive('execute_command').never() module.compact_segments( - repository_path='repo', storage_config={}, local_borg_version='1.2.3', dry_run=True + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + dry_run=True, ) @@ -66,6 +82,7 @@ def test_compact_segments_with_local_path_calls_borg_via_local_path(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -79,6 +96,7 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -92,6 +110,7 @@ def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -105,6 +124,7 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), cleanup_commits=True, ) @@ -118,6 +138,7 @@ def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): repository_path='repo', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), threshold=20, ) @@ -132,6 +153,20 @@ def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(COMPACT_COMMAND + ('--log-json', 'repo'), logging.INFO) + + module.compact_segments( + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -145,6 +180,7 @@ def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): repository_path='repo', storage_config=storage_config, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -157,4 +193,5 @@ def test_compact_segments_with_extra_borg_options_calls_borg_with_extra_options( repository_path='repo', storage_config={'extra_borg_options': {'compact': '--extra --options'}}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 3728a0b..e0462e2 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -492,6 +492,7 @@ def test_create_archive_calls_borg_with_parameters(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -535,6 +536,7 @@ def test_create_archive_calls_borg_with_environment(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -580,6 +582,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns_including_convert }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -625,6 +628,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -668,6 +672,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -708,6 +713,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, ) @@ -752,6 +758,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -792,6 +799,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, ) @@ -835,6 +843,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -880,6 +889,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stats=True, ) @@ -923,6 +933,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte }, storage_config={'checkpoint_interval': 600}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -965,6 +976,7 @@ def test_create_archive_with_checkpoint_volume_calls_borg_with_checkpoint_volume }, storage_config={'checkpoint_volume': 1024}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1007,6 +1019,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param }, storage_config={'chunker_params': '1,2,3,4'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1049,6 +1062,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters( }, storage_config={'compression': 'rle'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1097,6 +1111,7 @@ def test_create_archive_with_upload_rate_limit_calls_borg_with_upload_ratelimit_ }, storage_config={'upload_rate_limit': 100}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1142,6 +1157,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1185,6 +1201,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1234,6 +1251,7 @@ def test_create_archive_with_numeric_ids_calls_borg_with_numeric_ids_parameter( }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1287,6 +1305,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1342,6 +1361,7 @@ def test_create_archive_with_basic_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1396,6 +1416,7 @@ def test_create_archive_with_atime_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1450,6 +1471,7 @@ def test_create_archive_with_flags_option_calls_borg_with_corresponding_paramete }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1493,6 +1515,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters( }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1535,6 +1558,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -1578,6 +1602,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters( }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -1621,6 +1646,50 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters(): }, storage_config={'umask': 740}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_create_archive_with_log_json_calls_borg_with_log_json_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([]) + flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar')) + flexmock(module).should_receive('map_directories_to_devices').and_return({}) + flexmock(module).should_receive('expand_directories').and_return(()) + flexmock(module).should_receive('pattern_root_directories').and_return([]) + flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError) + flexmock(module).should_receive('expand_home_directories').and_return(()) + flexmock(module).should_receive('write_pattern_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module).should_receive('ensure_files_readable') + flexmock(module).should_receive('make_pattern_flags').and_return(()) + flexmock(module).should_receive('make_exclude_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + (f'repo::{DEFAULT_ARCHIVE_NAME}',) + ) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'create', '--log-json') + REPO_ARCHIVE_WITH_PATHS, + output_log_level=logging.INFO, + output_file=None, + borg_local_path='borg', + working_directory=None, + extra_environment=None, + ) + + module.create_archive( + dry_run=False, + repository_path='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -1663,6 +1732,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters(): }, storage_config={'lock_wait': 5}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -1705,6 +1775,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stats=True, ) @@ -1748,6 +1819,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_answer_out }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), list_files=True, ) @@ -1797,6 +1869,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -1840,6 +1913,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -1900,6 +1974,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, stream_processes=processes, ) @@ -1964,6 +2039,7 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2031,6 +2107,7 @@ def test_create_archive_with_stream_processes_adds_special_files_to_excludes(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2095,6 +2172,7 @@ def test_create_archive_with_stream_processes_and_read_special_does_not_add_spec }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2135,6 +2213,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, ) @@ -2177,6 +2256,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter() }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), json=True, stats=True, ) @@ -2224,6 +2304,7 @@ def test_create_archive_with_source_directories_glob_expands(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2267,6 +2348,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2309,6 +2391,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories(): }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2351,6 +2434,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): }, storage_config={'archive_name_format': 'ARCHIVE_NAME'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2394,6 +2478,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): }, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2437,6 +2522,7 @@ def test_create_archive_with_repository_accepts_borg_placeholders(): }, storage_config={'archive_name_format': 'Documents_{hostname}-{now}'}, # noqa: FS003 local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2479,6 +2565,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options(): }, storage_config={'extra_borg_options': {'create': '--extra --options'}}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -2539,6 +2626,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes_and_read }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), stream_processes=processes, ) @@ -2564,6 +2652,7 @@ def test_create_archive_with_non_existent_directory_and_source_directories_must_ }, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index 92776dd..5fb7bff 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -38,6 +38,7 @@ def test_export_tar_archive_calls_borg_with_path_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -59,6 +60,7 @@ def test_export_tar_archive_calls_borg_with_local_path_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', ) @@ -81,6 +83,7 @@ def test_export_tar_archive_calls_borg_with_remote_path_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -103,6 +106,27 @@ def test_export_tar_archive_calls_borg_with_umask_parameters(): destination_path='test.tar', storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_export_tar_archive_calls_borg_with_log_json_parameter(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + insert_execute_command_mock(('borg', 'export-tar', '--log-json', 'repo::archive', 'test.tar')) + + module.export_tar_archive( + dry_run=False, + repository_path='repo', + archive='archive', + paths=None, + destination_path='test.tar', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -124,6 +148,7 @@ def test_export_tar_archive_calls_borg_with_lock_wait_parameters(): destination_path='test.tar', storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -144,6 +169,7 @@ def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -166,6 +192,7 @@ def test_export_tar_archive_with_log_debug_calls_borg_with_debug_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -185,6 +212,7 @@ def test_export_tar_archive_calls_borg_with_dry_run_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -206,6 +234,7 @@ def test_export_tar_archive_calls_borg_with_tar_filter_parameters(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), tar_filter='bzip2', ) @@ -229,6 +258,7 @@ def test_export_tar_archive_calls_borg_with_list_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), list_files=True, ) @@ -251,6 +281,7 @@ def test_export_tar_archive_calls_borg_with_strip_components_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components=5, ) @@ -271,6 +302,7 @@ def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): destination_path='test.tar', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -290,4 +322,5 @@ def test_export_tar_archive_calls_borg_with_stdout_destination_path(): destination_path='-', storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index 26fd738..a4032f6 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -25,7 +25,11 @@ def test_extract_last_archive_dry_run_calls_borg_with_last_archive(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -34,7 +38,11 @@ def test_extract_last_archive_dry_run_without_any_archives_should_not_raise(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(('repo',)) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -47,7 +55,11 @@ def test_extract_last_archive_dry_run_with_log_info_calls_borg_with_info_paramet ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -62,7 +74,11 @@ def test_extract_last_archive_dry_run_with_log_debug_calls_borg_with_debug_param ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=None + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=None, ) @@ -76,13 +92,14 @@ def test_extract_last_archive_dry_run_calls_borg_via_local_path(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), repository_path='repo', lock_wait=None, local_path='borg1', ) -def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters(): +def test_extract_last_archive_dry_run_calls_borg_with_remote_path_flags(): flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') insert_execute_command_mock( ('borg', 'extract', '--dry-run', '--remote-path', 'borg1', 'repo::archive') @@ -94,13 +111,30 @@ def test_extract_last_archive_dry_run_calls_borg_with_remote_path_parameters(): module.extract_last_archive_dry_run( storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), repository_path='repo', lock_wait=None, remote_path='borg1', ) -def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters(): +def test_extract_last_archive_dry_run_calls_borg_with_log_json_flag(): + flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') + insert_execute_command_mock(('borg', 'extract', '--dry-run', '--log-json', 'repo::archive')) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + + module.extract_last_archive_dry_run( + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + repository_path='repo', + lock_wait=None, + ) + + +def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_flags(): flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive') insert_execute_command_mock( ('borg', 'extract', '--dry-run', '--lock-wait', '5', 'repo::archive') @@ -110,17 +144,24 @@ def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters(): ) module.extract_last_archive_dry_run( - storage_config={}, local_borg_version='1.2.3', repository_path='repo', lock_wait=5 + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + repository_path='repo', + lock_wait=5, ) -def test_extract_archive_calls_borg_with_path_parameters(): +def test_extract_archive_calls_borg_with_path_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock(('borg', 'extract', 'repo::archive', 'path1', 'path2')) flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -130,16 +171,20 @@ def test_extract_archive_calls_borg_with_path_parameters(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_calls_borg_with_remote_path_parameters(): +def test_extract_archive_calls_borg_with_remote_path_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock(('borg', 'extract', '--remote-path', 'borg1', 'repo::archive')) flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -149,6 +194,7 @@ def test_extract_archive_calls_borg_with_remote_path_parameters(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -167,6 +213,9 @@ def test_extract_archive_calls_borg_with_numeric_ids_parameter(feature_available flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -176,16 +225,20 @@ def test_extract_archive_calls_borg_with_numeric_ids_parameter(feature_available location_config={'numeric_ids': True}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_calls_borg_with_umask_parameters(): +def test_extract_archive_calls_borg_with_umask_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock(('borg', 'extract', '--umask', '0770', 'repo::archive')) flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -195,17 +248,41 @@ def test_extract_archive_calls_borg_with_umask_parameters(): location_config={}, storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_calls_borg_with_lock_wait_parameters(): +def test_extract_archive_calls_borg_with_log_json_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') - insert_execute_command_mock(('borg', 'extract', '--lock-wait', '5', 'repo::archive')) + insert_execute_command_mock(('borg', 'extract', '--log-json', 'repo::archive')) flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + module.extract_archive( + dry_run=False, + repository='repo', + archive='archive', + paths=None, + location_config={}, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + ) + + +def test_extract_archive_calls_borg_with_lock_wait_flags(): + flexmock(module.os.path).should_receive('abspath').and_return('repo') + insert_execute_command_mock(('borg', 'extract', '--lock-wait', '5', 'repo::archive')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') + module.extract_archive( dry_run=False, repository='repo', @@ -214,6 +291,7 @@ def test_extract_archive_calls_borg_with_lock_wait_parameters(): location_config={}, storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -225,6 +303,9 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -234,10 +315,11 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) -def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters(): +def test_extract_archive_with_log_debug_calls_borg_with_debug_flags(): flexmock(module.os.path).should_receive('abspath').and_return('repo') insert_execute_command_mock( ('borg', 'extract', '--debug', '--list', '--show-rc', 'repo::archive') @@ -247,6 +329,9 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -256,6 +341,7 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -266,6 +352,9 @@ def test_extract_archive_calls_borg_with_dry_run_parameter(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=True, @@ -275,6 +364,7 @@ def test_extract_archive_calls_borg_with_dry_run_parameter(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -285,6 +375,9 @@ def test_extract_archive_calls_borg_with_destination_path(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -294,6 +387,7 @@ def test_extract_archive_calls_borg_with_destination_path(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), destination_path='/dest', ) @@ -305,6 +399,9 @@ def test_extract_archive_calls_borg_with_strip_components(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -314,6 +411,7 @@ def test_extract_archive_calls_borg_with_strip_components(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components=5, ) @@ -335,6 +433,9 @@ def test_extract_archive_calls_borg_with_strip_components_calculated_from_all(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -344,6 +445,7 @@ def test_extract_archive_calls_borg_with_strip_components_calculated_from_all(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components='all', ) @@ -354,6 +456,9 @@ def test_extract_archive_with_strip_components_all_and_no_paths_raises(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') flexmock(module).should_receive('execute_command').never() with pytest.raises(ValueError): @@ -365,6 +470,7 @@ def test_extract_archive_with_strip_components_all_and_no_paths_raises(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), strip_components='all', ) @@ -382,6 +488,9 @@ def test_extract_archive_calls_borg_with_progress_parameter(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -391,6 +500,7 @@ def test_extract_archive_calls_borg_with_progress_parameter(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, ) @@ -407,6 +517,7 @@ def test_extract_archive_with_progress_and_extract_to_stdout_raises(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), progress=True, extract_to_stdout=True, ) @@ -427,6 +538,9 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') assert ( module.extract_archive( @@ -437,6 +551,7 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), extract_to_stdout=True, ) == process @@ -455,6 +570,9 @@ def test_extract_archive_skips_abspath_for_remote_repository(): flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( ('server:repo::archive',) ) + flexmock(module.borgmatic.config.validate).should_receive( + 'normalize_repository_path' + ).and_return('repo') module.extract_archive( dry_run=False, @@ -464,4 +582,5 @@ def test_extract_archive_skips_abspath_for_remote_repository(): location_config={}, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index d7bf731..e7b126a 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -29,6 +29,7 @@ def test_display_archives_info_calls_borg_with_parameters(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -54,6 +55,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -78,6 +80,7 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) @@ -106,6 +109,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -130,6 +134,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) @@ -155,6 +160,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None), ) @@ -182,6 +188,7 @@ def test_display_archives_info_with_archive_calls_borg_with_match_archives_param repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive='archive', json=False, prefix=None, match_archives=None), ) @@ -207,6 +214,7 @@ def test_display_archives_info_with_local_path_calls_borg_via_local_path(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), local_path='borg1', ) @@ -236,11 +244,41 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), remote_path='borg1', ) +def test_display_archives_info_with_log_json_calls_borg_with_log_json_parameters(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args('log-json', True).and_return( + ('--log-json',) + ) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '2.3.4' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'info', '--log-json', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_archives_info( + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=True), + info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), + ) + + def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_parameters(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -266,6 +304,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete repository_path='repo', storage_config=storage_config, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -294,6 +333,7 @@ def test_display_archives_info_transforms_prefix_into_match_archives_parameters( repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix='foo'), ) @@ -322,6 +362,7 @@ def test_display_archives_info_prefers_prefix_over_archive_name_format(): repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix='foo'), ) @@ -347,6 +388,7 @@ def test_display_archives_info_transforms_archive_name_format_into_match_archive repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -375,6 +417,7 @@ def test_display_archives_with_match_archives_option_calls_borg_with_match_archi 'match_archives': 'sh:foo-*', }, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives=None), ) @@ -400,6 +443,7 @@ def test_display_archives_with_match_archives_flag_calls_borg_with_match_archive repository_path='repo', storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'), ) @@ -429,6 +473,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=flexmock( archive=None, json=False, prefix=None, match_archives=None, **{argument_name: 'value'} ), @@ -480,5 +525,6 @@ def test_display_archives_info_with_date_based_matching_calls_borg_with_date_bas repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), info_arguments=info_arguments, ) diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 0a7db4c..4e3a5f7 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -20,6 +20,7 @@ def test_make_list_command_includes_log_info(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--info', 'repo') @@ -36,6 +37,7 @@ def test_make_list_command_includes_json_but_not_info(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -52,6 +54,7 @@ def test_make_list_command_includes_log_debug(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--debug', '--show-rc', 'repo') @@ -68,6 +71,7 @@ def test_make_list_command_includes_json_but_not_debug(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -83,11 +87,28 @@ def test_make_list_command_includes_json(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') +def test_make_list_command_includes_log_json(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return(('--log-json',)) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_list_command( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=True), + ) + + assert command == ('borg', 'list', '--log-json', 'repo') + + def test_make_list_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') @@ -100,6 +121,7 @@ def test_make_list_command_includes_lock_wait(): storage_config={'lock_wait': 5}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--lock-wait', '5', 'repo') @@ -117,6 +139,7 @@ def test_make_list_command_includes_archive(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=None, json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', 'repo::archive') @@ -134,6 +157,7 @@ def test_make_list_command_includes_archive_and_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive='archive', paths=['var/lib'], json=False), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', 'repo::archive', 'var/lib') @@ -149,6 +173,7 @@ def test_make_list_command_includes_local_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), local_path='borg2', ) @@ -156,9 +181,13 @@ def test_make_list_command_includes_local_path(): def test_make_list_command_includes_remote_path(): - flexmock(module.flags).should_receive('make_flags').and_return( - ('--remote-path', 'borg2') - ).and_return(()) + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args( + 'remote-path', 'borg2' + ).and_return(('--remote-path', 'borg2')) + flexmock(module.flags).should_receive('make_flags').with_args('log-json', True).and_return( + ('--log-json') + ) flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) @@ -167,6 +196,7 @@ def test_make_list_command_includes_remote_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False), + global_arguments=flexmock(log_json=False), remote_path='borg2', ) @@ -183,6 +213,7 @@ def test_make_list_command_includes_short(): storage_config={}, local_borg_version='1.2.3', list_arguments=flexmock(archive=None, paths=None, json=False, short=True), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--short', 'repo') @@ -221,6 +252,7 @@ def test_make_list_command_includes_additional_flags(argument_name): format=None, **{argument_name: 'value'}, ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo') @@ -263,10 +295,11 @@ def test_capture_archive_listing_does_not_raise(): archive='archive', storage_config=flexmock(), local_borg_version=flexmock(), + global_arguments=flexmock(log_json=False), ) -def test_list_archive_calls_borg_with_parameters(): +def test_list_archive_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.logger).answer = lambda message: None @@ -281,6 +314,7 @@ def test_list_archive_calls_borg_with_parameters(): first=None, last=None, ) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -288,6 +322,7 @@ def test_list_archive_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', 'repo::archive')) @@ -305,6 +340,7 @@ def test_list_archive_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, ) @@ -322,6 +358,7 @@ def test_list_archive_with_archive_and_json_errors(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -340,6 +377,7 @@ def test_list_archive_calls_borg_with_local_path(): first=None, last=None, ) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -347,6 +385,7 @@ def test_list_archive_calls_borg_with_local_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg2', remote_path=None, ).and_return(('borg2', 'list', 'repo::archive')) @@ -364,6 +403,7 @@ def test_list_archive_calls_borg_with_local_path(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg2', ) @@ -413,6 +453,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -431,6 +472,7 @@ def test_list_archive_calls_borg_with_archive(): first=None, last=None, ) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_list_command').with_args( @@ -438,6 +480,7 @@ def test_list_archive_calls_borg_with_archive(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', 'repo::archive')) @@ -455,6 +498,7 @@ def test_list_archive_calls_borg_with_archive(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=global_arguments, ) @@ -485,6 +529,7 @@ def test_list_archive_without_archive_delegates_to_list_repository(): storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -515,6 +560,7 @@ def test_list_archive_with_borg_features_without_archive_delegates_to_list_repos storage_config={}, local_borg_version='1.2.3', list_arguments=list_arguments, + global_arguments=flexmock(log_json=False), ) @@ -534,6 +580,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.logger).answer = lambda message: None + global_arguments = flexmock(log_json=False) default_filter_flags = { 'prefix': None, 'match_archives': None, @@ -553,6 +600,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( list_arguments=argparse.Namespace( archive='archive', paths=None, json=False, find_paths=None, **default_filter_flags ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', 'repo::archive')) @@ -572,6 +620,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( list_arguments=argparse.Namespace( archive='archive', paths=None, json=False, find_paths=None, **altered_filter_flags ), + global_arguments=global_arguments, ) @@ -600,6 +649,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes } altered_filter_flags = {**default_filter_flags, **{archive_filter_flag: 'foo'}} glob_paths = ('**/*foo.txt*/**',) + global_arguments = flexmock(log_json=False) flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.rlist).should_receive('make_rlist_command').with_args( @@ -609,6 +659,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes rlist_arguments=argparse.Namespace( repository='repo', short=True, format=None, json=None, **altered_filter_flags ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'rlist', '--repo', 'repo')) @@ -632,6 +683,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes find_paths=['foo.txt'], **default_filter_flags, ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', '--repo', 'repo', 'archive1')) @@ -650,6 +702,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes find_paths=['foo.txt'], **default_filter_flags, ), + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'list', '--repo', 'repo', 'archive2')) @@ -683,4 +736,5 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes find_paths=['foo.txt'], **altered_filter_flags, ), + global_arguments=global_arguments, ) diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 05ff72e..bd93cf3 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -28,6 +28,7 @@ def test_mount_archive_calls_borg_with_required_flags(): mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -50,6 +51,7 @@ def test_mount_archive_with_borg_features_calls_borg_with_repository_and_match_a mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -67,6 +69,7 @@ def test_mount_archive_without_archive_calls_borg_with_repository_flags_only(): mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -86,6 +89,7 @@ def test_mount_archive_calls_borg_with_path_flags(): mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -105,6 +109,7 @@ def test_mount_archive_calls_borg_with_remote_path_flags(): mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', ) @@ -123,6 +128,25 @@ def test_mount_archive_calls_borg_with_umask_flags(): mount_arguments=mount_arguments, storage_config={'umask': '0770'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) + + +def test_mount_archive_calls_borg_with_log_json_flags(): + flexmock(module.feature).should_receive('available').and_return(False) + flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( + ('repo::archive',) + ) + insert_execute_command_mock(('borg', 'mount', '--log-json', 'repo::archive', '/mnt')) + + mount_arguments = flexmock(mount_point='/mnt', options=None, paths=None, foreground=False) + module.mount_archive( + repository_path='repo', + archive='archive', + mount_arguments=mount_arguments, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), ) @@ -140,6 +164,7 @@ def test_mount_archive_calls_borg_with_lock_wait_flags(): mount_arguments=mount_arguments, storage_config={'lock_wait': '5'}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -158,6 +183,7 @@ def test_mount_archive_with_log_info_calls_borg_with_info_parameter(): mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -176,6 +202,7 @@ def test_mount_archive_with_log_debug_calls_borg_with_debug_flags(): mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -199,6 +226,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter(): mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) @@ -218,18 +246,30 @@ def test_mount_archive_calls_borg_with_options_flags(): mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) def test_mount_archive_with_date_based_matching_calls_borg_with_date_based_flags(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_repository_flags').and_return( + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( ( - '--repo', - 'repo', + '--newer', + '1d', + '--newest', + '1y', + '--older', + '1m', + '--oldest', + '1w', + '--match-archives', + None, ) ) - insert_execute_command_mock( + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( ( 'borg', 'mount', @@ -241,12 +281,14 @@ def test_mount_archive_with_date_based_matching_calls_borg_with_date_based_flags '1m', '--oldest', '1w', - '--repo', - 'repo', '--match-archives', None, + '--repo', + 'repo', '/mnt', - ) + ), + borg_local_path='borg', + extra_environment=None, ) mount_arguments = flexmock( @@ -265,4 +307,5 @@ def test_mount_archive_with_date_based_matching_calls_borg_with_date_based_flags mount_arguments=mount_arguments, storage_config={}, local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 66cb7cd..584d0d5 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -110,7 +110,7 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): PRUNE_COMMAND = ('borg', 'prune', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3') -def test_prune_archives_calls_borg_with_parameters(): +def test_prune_archives_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -124,11 +124,12 @@ def test_prune_archives_calls_borg_with_parameters(): storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) -def test_prune_archives_with_log_info_calls_borg_with_info_parameter(): +def test_prune_archives_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -143,11 +144,12 @@ def test_prune_archives_with_log_info_calls_borg_with_info_parameter(): dry_run=False, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) -def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter(): +def test_prune_archives_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -162,11 +164,12 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_parameter(): dry_run=False, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) -def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): +def test_prune_archives_with_dry_run_calls_borg_with_dry_run_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -180,6 +183,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_parameter(): dry_run=True, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) @@ -198,12 +202,13 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), local_path='borg1', prune_arguments=prune_arguments, ) -def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -217,12 +222,13 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_parameters( storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), remote_path='borg1', prune_arguments=prune_arguments, ) -def test_prune_archives_with_stats_calls_borg_with_stats_parameter_and_answer_output_log_level(): +def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -236,11 +242,12 @@ def test_prune_archives_with_stats_calls_borg_with_stats_parameter_and_answer_ou storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) -def test_prune_archives_with_files_calls_borg_with_list_parameter_and_answer_output_log_level(): +def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -254,11 +261,12 @@ def test_prune_archives_with_files_calls_borg_with_list_parameter_and_answer_out storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) -def test_prune_archives_with_umask_calls_borg_with_umask_parameters(): +def test_prune_archives_with_umask_calls_borg_with_umask_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'umask': '077'} @@ -273,11 +281,31 @@ def test_prune_archives_with_umask_calls_borg_with_umask_parameters(): storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) -def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_prune_archives_with_log_json_calls_borg_with_log_json_flag(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + insert_execute_command_mock(PRUNE_COMMAND + ('--log-json', 'repo'), logging.INFO) + + prune_arguments = flexmock(stats=False, list_archives=False) + module.prune_archives( + dry_run=False, + repository_path='repo', + storage_config={}, + retention_config=flexmock(), + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + prune_arguments=prune_arguments, + ) + + +def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} @@ -292,6 +320,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_parameters(): storage_config=storage_config, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) @@ -310,6 +339,7 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): storage_config={'extra_borg_options': {'prune': '--extra --options'}}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) @@ -317,12 +347,51 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) - insert_execute_command_mock( - PRUNE_COMMAND - + ('--newer', '1d', '--newest', '1y', '--older', '1m', '--oldest', '1w', 'repo'), - logging.INFO, + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return( + ( + '--newer', + '1d', + '--newest', + '1y', + '--older', + '1m', + '--oldest', + '1w', + '--match-archives', + None, + ) + ) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ( + 'borg', + 'prune', + '--keep-daily', + '1', + '--keep-weekly', + '2', + '--keep-monthly', + '3', + '--newer', + '1d', + '--newest', + '1y', + '--older', + '1m', + '--oldest', + '1w', + '--match-archives', + None, + '--repo', + 'repo', + ), + output_log_level=logging.INFO, + borg_local_path='borg', + extra_environment=None, ) prune_arguments = flexmock( @@ -334,5 +403,6 @@ def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flag storage_config={}, retention_config=flexmock(), local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) diff --git a/tests/unit/borg/test_rcreate.py b/tests/unit/borg/test_rcreate.py index 4da04df..2f71a8f 100644 --- a/tests/unit/borg/test_rcreate.py +++ b/tests/unit/borg/test_rcreate.py @@ -48,6 +48,7 @@ def test_create_repository_calls_borg_with_flags(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -68,6 +69,7 @@ def test_create_repository_with_dry_run_skips_borg_call(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -92,6 +94,7 @@ def test_create_repository_raises_for_borg_rcreate_error(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -111,6 +114,7 @@ def test_create_repository_skips_creation_when_repository_already_exists(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -126,6 +130,7 @@ def test_create_repository_raises_for_unknown_rinfo_command_error(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -146,6 +151,7 @@ def test_create_repository_with_source_repository_calls_borg_with_other_repo_fla repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', source_repository='other.borg', ) @@ -167,6 +173,7 @@ def test_create_repository_with_copy_crypt_key_calls_borg_with_copy_crypt_key_fl repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', copy_crypt_key=True, ) @@ -188,6 +195,7 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', append_only=True, ) @@ -209,6 +217,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', storage_quota='5G', ) @@ -230,6 +239,7 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', make_parent_dirs=True, ) @@ -252,6 +262,7 @@ def test_create_repository_with_log_info_calls_borg_with_info_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -273,6 +284,49 @@ def test_create_repository_with_log_debug_calls_borg_with_debug_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), + encryption_mode='repokey', + ) + + +def test_create_repository_with_log_json_calls_borg_with_log_json_flag(): + insert_rinfo_command_not_found_mock() + insert_rcreate_command_mock(RCREATE_COMMAND + ('--log-json', '--repo', 'repo')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + + module.create_repository( + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=True), + encryption_mode='repokey', + ) + + +def test_create_repository_with_lock_wait_calls_borg_with_lock_wait_flag(): + insert_rinfo_command_not_found_mock() + insert_rcreate_command_mock(RCREATE_COMMAND + ('--lock-wait', '5', '--repo', 'repo')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + + module.create_repository( + dry_run=False, + repository_path='repo', + storage_config={'lock_wait': 5}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) @@ -293,6 +347,7 @@ def test_create_repository_with_local_path_calls_borg_via_local_path(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', local_path='borg1', ) @@ -314,6 +369,7 @@ def test_create_repository_with_remote_path_calls_borg_with_remote_path_flag(): repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', remote_path='borg1', ) @@ -335,5 +391,6 @@ def test_create_repository_with_extra_borg_options_calls_borg_with_extra_options repository_path='repo', storage_config={'extra_borg_options': {'rcreate': '--extra --options'}}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), encryption_mode='repokey', ) diff --git a/tests/unit/borg/test_rinfo.py b/tests/unit/borg/test_rinfo.py index 979b253..a6e3f08 100644 --- a/tests/unit/borg/test_rinfo.py +++ b/tests/unit/borg/test_rinfo.py @@ -7,7 +7,7 @@ from borgmatic.borg import rinfo as module from ..test_verbosity import insert_logging_mock -def test_display_repository_info_calls_borg_with_parameters(): +def test_display_repository_info_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -30,6 +30,7 @@ def test_display_repository_info_calls_borg_with_parameters(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) @@ -51,10 +52,11 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_ storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) -def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): +def test_display_repository_info_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -77,6 +79,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_parameter(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) @@ -102,12 +105,13 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), + global_arguments=flexmock(log_json=False), ) assert json_output == '[]' -def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter(): +def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -131,6 +135,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_parameter( storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) @@ -156,12 +161,13 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), + global_arguments=flexmock(log_json=False), ) assert json_output == '[]' -def test_display_repository_info_with_json_calls_borg_with_json_parameter(): +def test_display_repository_info_with_json_calls_borg_with_json_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -182,6 +188,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_parameter(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=True), + global_arguments=flexmock(log_json=False), ) assert json_output == '[]' @@ -210,11 +217,12 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), local_path='borg1', ) -def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.feature).should_receive('available').and_return(True) @@ -237,11 +245,39 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_pa storage_config={}, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), remote_path='borg1', ) -def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'rinfo', '--log-json', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + borg_local_path='borg', + extra_environment=None, + ) + + module.display_repository_info( + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=True), + ) + + +def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER storage_config = {'lock_wait': 5} @@ -265,4 +301,5 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_parame storage_config=storage_config, local_borg_version='2.3.4', rinfo_arguments=flexmock(json=False), + global_arguments=flexmock(log_json=False), ) diff --git a/tests/unit/borg/test_rlist.py b/tests/unit/borg/test_rlist.py index b6fadd1..76bda98 100644 --- a/tests/unit/borg/test_rlist.py +++ b/tests/unit/borg/test_rlist.py @@ -20,12 +20,18 @@ def test_resolve_archive_name_passes_through_non_latest_archive_name(): archive = 'myhost-2030-01-01T14:41:17.647620' assert ( - module.resolve_archive_name('repo', archive, storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + archive, + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == archive ) -def test_resolve_archive_name_calls_borg_with_parameters(): +def test_resolve_archive_name_calls_borg_with_flags(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -34,12 +40,18 @@ def test_resolve_archive_name_calls_borg_with_parameters(): ).and_return(expected_archive + '\n') assert ( - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == expected_archive ) -def test_resolve_archive_name_with_log_info_calls_borg_without_info_parameter(): +def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -49,12 +61,18 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_parameter(): insert_logging_mock(logging.INFO) assert ( - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == expected_archive ) -def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_parameter(): +def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -64,7 +82,13 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_parameter( insert_logging_mock(logging.DEBUG) assert ( - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) == expected_archive ) @@ -79,13 +103,18 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): assert ( module.resolve_archive_name( - 'repo', 'latest', storage_config={}, local_borg_version='1.2.3', local_path='borg1' + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + local_path='borg1', ) == expected_archive ) -def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -95,7 +124,12 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param assert ( module.resolve_archive_name( - 'repo', 'latest', storage_config={}, local_borg_version='1.2.3', remote_path='borg1' + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + remote_path='borg1', ) == expected_archive ) @@ -109,10 +143,37 @@ def test_resolve_archive_name_without_archives_raises(): ).and_return('') with pytest.raises(ValueError): - module.resolve_archive_name('repo', 'latest', storage_config={}, local_borg_version='1.2.3') + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + ) -def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags(): + expected_archive = 'archive-name' + + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ('borg', 'list', '--log-json') + BORG_LIST_LATEST_ARGUMENTS, + extra_environment=None, + ).and_return(expected_archive + '\n') + + assert ( + module.resolve_archive_name( + 'repo', + 'latest', + storage_config={}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=True), + ) + == expected_archive + ) + + +def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags(): expected_archive = 'archive-name' flexmock(module.environment).should_receive('make_environment') @@ -123,7 +184,11 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter assert ( module.resolve_archive_name( - 'repo', 'latest', storage_config={'lock_wait': 'okay'}, local_borg_version='1.2.3' + 'repo', + 'latest', + storage_config={'lock_wait': 'okay'}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), ) == expected_archive ) @@ -145,6 +210,7 @@ def test_make_rlist_command_includes_log_info(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--info', 'repo') @@ -166,6 +232,7 @@ def test_make_rlist_command_includes_json_but_not_info(): rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -187,6 +254,7 @@ def test_make_rlist_command_includes_log_debug(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--debug', '--show-rc', 'repo') @@ -208,6 +276,7 @@ def test_make_rlist_command_includes_json_but_not_debug(): rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') @@ -228,11 +297,35 @@ def test_make_rlist_command_includes_json(): rlist_arguments=flexmock( archive=None, paths=None, json=True, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--json', 'repo') +def test_make_rlist_command_includes_log_json(): + flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( + ('--log-json',) + ).and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').with_args( + None, None, '1.2.3' + ).and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + + command = module.make_rlist_command( + repository_path='repo', + storage_config={}, + local_borg_version='1.2.3', + rlist_arguments=flexmock( + archive=None, paths=None, json=False, prefix=None, match_archives=None + ), + global_arguments=flexmock(log_json=True), + ) + + assert command == ('borg', 'list', '--log-json', 'repo') + + def test_make_rlist_command_includes_lock_wait(): flexmock(module.flags).should_receive('make_flags').and_return(()).and_return( ('--lock-wait', '5') @@ -250,6 +343,7 @@ def test_make_rlist_command_includes_lock_wait(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--lock-wait', '5', 'repo') @@ -270,6 +364,7 @@ def test_make_rlist_command_includes_local_path(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), local_path='borg2', ) @@ -293,6 +388,7 @@ def test_make_rlist_command_includes_remote_path(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), remote_path='borg2', ) @@ -314,6 +410,7 @@ def test_make_rlist_command_transforms_prefix_into_match_archives(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') @@ -332,6 +429,7 @@ def test_make_rlist_command_prefers_prefix_over_archive_name_format(): storage_config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 local_borg_version='1.2.3', rlist_arguments=flexmock(archive=None, paths=None, json=False, prefix='foo'), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'sh:foo*', 'repo') @@ -352,6 +450,7 @@ def test_make_rlist_command_transforms_archive_name_format_into_match_archives() rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'sh:bar-*', 'repo') @@ -372,6 +471,7 @@ def test_make_rlist_command_includes_short(): rlist_arguments=flexmock( archive=None, paths=None, json=False, prefix=None, match_archives=None, short=True ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--short', 'repo') @@ -413,12 +513,13 @@ def test_make_rlist_command_includes_additional_flags(argument_name): format=None, **{argument_name: 'value'}, ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo') -def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_parameters(): +def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_flags(): flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( None, None, '1.2.3' @@ -444,15 +545,17 @@ def test_make_rlist_command_with_match_archives_calls_borg_with_match_archives_p find_paths=None, format=None, ), + global_arguments=flexmock(log_json=False), ) assert command == ('borg', 'list', '--match-archives', 'foo-*', 'repo') -def test_list_repository_calls_borg_with_parameters(): +def test_list_repository_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER rlist_arguments = argparse.Namespace(json=False) + global_arguments = flexmock() flexmock(module.feature).should_receive('available').and_return(False) flexmock(module).should_receive('make_rlist_command').with_args( @@ -460,6 +563,7 @@ def test_list_repository_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'rlist', 'repo')) @@ -476,6 +580,7 @@ def test_list_repository_calls_borg_with_parameters(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, ) @@ -483,6 +588,7 @@ def test_list_repository_with_json_returns_borg_output(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER rlist_arguments = argparse.Namespace(json=True) + global_arguments = flexmock() json_output = flexmock() flexmock(module.feature).should_receive('available').and_return(False) @@ -491,6 +597,7 @@ def test_list_repository_with_json_returns_borg_output(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, local_path='borg', remote_path=None, ).and_return(('borg', 'rlist', 'repo')) @@ -503,6 +610,7 @@ def test_list_repository_with_json_returns_borg_output(): storage_config={}, local_borg_version='1.2.3', rlist_arguments=rlist_arguments, + global_arguments=global_arguments, ) == json_output ) @@ -533,6 +641,7 @@ def test_make_rlist_command_with_date_based_matching_calls_borg_with_date_based_ older='1m', oldest='1w', ), + global_arguments=flexmock(log_json=False), ) assert command == ( diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index fd6cb19..6cd9530 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -32,6 +32,7 @@ def test_transfer_archives_calls_borg_with_flags(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -62,6 +63,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -89,6 +91,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -117,6 +120,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -146,6 +150,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): transfer_arguments=flexmock( archive='archive', progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -175,6 +180,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -204,6 +210,7 @@ def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archiv transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -231,6 +238,7 @@ def test_transfer_archives_with_local_path_calls_borg_via_local_path(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), local_path='borg2', ) @@ -262,10 +270,42 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), remote_path='borg2', ) +def test_transfer_archives_with_log_json_calls_borg_with_log_json_flags(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args('log-json', True).and_return( + ('--log-json',) + ) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'transfer', '--log-json', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + output_file=None, + borg_local_path='borg', + extra_environment=None, + ) + + module.transfer_archives( + dry_run=False, + repository_path='repo', + storage_config={}, + local_borg_version='2.3.4', + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), + global_arguments=flexmock(log_json=True), + ) + + def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER @@ -294,6 +334,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -321,6 +362,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): transfer_arguments=flexmock( archive=None, progress=True, match_archives=None, source_repository=None ), + global_arguments=flexmock(log_json=False), ) @@ -356,6 +398,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): source_repository=None, **{argument_name: 'value'}, ), + global_arguments=flexmock(log_json=False), ) @@ -385,6 +428,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla transfer_arguments=flexmock( archive=None, progress=None, match_archives=None, source_repository='other' ), + global_arguments=flexmock(log_json=False), ) @@ -423,6 +467,7 @@ def test_transfer_archives_with_date_based_matching_calls_borg_with_date_based_f repository_path='repo', storage_config={}, local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), transfer_arguments=flexmock( archive=None, progress=None, diff --git a/tests/unit/commands/test_completions.py b/tests/unit/commands/test_completions.py new file mode 100644 index 0000000..12829d5 --- /dev/null +++ b/tests/unit/commands/test_completions.py @@ -0,0 +1,135 @@ +from argparse import Action +from collections import namedtuple +from typing import Tuple + +import pytest +from flexmock import flexmock + +from borgmatic.commands import completion as module + +OptionType = namedtuple('OptionType', ['file', 'choice', 'unknown_required']) +TestCase = Tuple[Action, OptionType] + +test_data = [ + (Action('--flag', 'flag'), OptionType(file=False, choice=False, unknown_required=False)), + *( + ( + Action('--flag', 'flag', metavar=metavar), + OptionType(file=True, choice=False, unknown_required=False), + ) + for metavar in ('FILENAME', 'PATH') + ), + ( + Action('--flag', dest='config_paths'), + OptionType(file=True, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', metavar='OTHER'), + OptionType(file=False, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', choices=['a', 'b']), + OptionType(file=False, choice=True, unknown_required=False), + ), + ( + Action('--flag', 'flag', choices=['a', 'b'], type=str), + OptionType(file=False, choice=True, unknown_required=True), + ), + ( + Action('--flag', 'flag', choices=None), + OptionType(file=False, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', required=True), + OptionType(file=False, choice=False, unknown_required=True), + ), + *( + ( + Action('--flag', 'flag', nargs=nargs), + OptionType(file=False, choice=False, unknown_required=True), + ) + for nargs in ('+', '*') + ), + *( + ( + Action('--flag', 'flag', metavar=metavar), + OptionType(file=False, choice=False, unknown_required=True), + ) + for metavar in ('PATTERN', 'KEYS', 'N') + ), + *( + ( + Action('--flag', 'flag', type=type, default=None), + OptionType(file=False, choice=False, unknown_required=True), + ) + for type in (int, str) + ), + ( + Action('--flag', 'flag', type=int, default=1), + OptionType(file=False, choice=False, unknown_required=False), + ), + ( + Action('--flag', 'flag', type=str, required=True, metavar='PATH'), + OptionType(file=True, choice=False, unknown_required=True), + ), + ( + Action('--flag', 'flag', type=str, required=True, metavar='PATH', default='/dev/null'), + OptionType(file=True, choice=False, unknown_required=True), + ), + ( + Action('--flag', 'flag', type=str, required=False, metavar='PATH', default='/dev/null'), + OptionType(file=True, choice=False, unknown_required=False), + ), +] + + +@pytest.mark.parametrize('action, option_type', test_data) +def test_has_file_options_detects_file_options(action: Action, option_type: OptionType): + assert module.has_file_options(action) == option_type.file + + +@pytest.mark.parametrize('action, option_type', test_data) +def test_has_choice_options_detects_choice_options(action: Action, option_type: OptionType): + assert module.has_choice_options(action) == option_type.choice + + +@pytest.mark.parametrize('action, option_type', test_data) +def test_has_unknown_required_param_options_detects_unknown_required_param_options( + action: Action, option_type: OptionType +): + assert module.has_unknown_required_param_options(action) == option_type.unknown_required + + +@pytest.mark.parametrize('action, option_type', test_data) +def test_has_exact_options_detects_exact_options(action: Action, option_type: OptionType): + assert module.has_exact_options(action) == (True in option_type) + + +@pytest.mark.parametrize('action, option_type', test_data) +def test_exact_options_completion_produces_reasonable_completions( + action: Action, option_type: OptionType +): + completion = module.exact_options_completion(action) + if True in option_type: + assert completion.startswith('\ncomplete -c borgmatic') + else: + assert completion == '' + + +def test_exact_options_completion_raises_for_unexpected_action(): + flexmock(module).should_receive('has_exact_options').and_return(True) + flexmock(module).should_receive('has_file_options').and_return(False) + flexmock(module).should_receive('has_choice_options').and_return(False) + flexmock(module).should_receive('has_unknown_required_param_options').and_return(False) + + with pytest.raises(ValueError): + module.exact_options_completion(Action('--unknown', dest='unknown')) + + +def test_dedent_strip_as_tuple_does_not_raise(): + module.dedent_strip_as_tuple( + ''' + a + b + ''' + ) diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index e2b9f98..e81f2b0 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -16,14 +16,13 @@ def test_schema_filename_finds_schema_path(): assert module.schema_filename() == schema_path -def test_schema_filename_with_missing_schema_path_raises(): +def test_schema_filename_with_missing_schema_path_in_package_still_finds_it_in_config_directory(): flexmock(module.importlib_metadata).should_receive('files').and_return( flexmock(match=lambda path: False, locate=lambda: None), flexmock(match=lambda path: False, locate=lambda: None), ) - with pytest.raises(FileNotFoundError): - assert module.schema_filename() + assert module.schema_filename().endswith('/borgmatic/config/schema.yaml') def test_format_json_error_path_element_formats_array_index():