feat: allow restoring to different port/host/username (#326).
Merge pull request #73 from diivi/feat/restore-with-different-hostname-port-username
This commit is contained in:
commit
68d90e1e40
14 changed files with 1222 additions and 65 deletions
|
@ -68,9 +68,11 @@ def restore_single_database(
|
||||||
archive_name,
|
archive_name,
|
||||||
hook_name,
|
hook_name,
|
||||||
database,
|
database,
|
||||||
|
connection_params,
|
||||||
): # pragma: no cover
|
): # pragma: no cover
|
||||||
'''
|
'''
|
||||||
Given (among other things) an archive name, a database hook name, and a configured database
|
Given (among other things) an archive name, a database hook name, the hostname,
|
||||||
|
port, username and password as connection params, and a configured database
|
||||||
configuration dict, restore that database from the archive.
|
configuration dict, restore that database from the archive.
|
||||||
'''
|
'''
|
||||||
logger.info(
|
logger.info(
|
||||||
|
@ -113,6 +115,7 @@ def restore_single_database(
|
||||||
location,
|
location,
|
||||||
global_arguments.dry_run,
|
global_arguments.dry_run,
|
||||||
extract_process,
|
extract_process,
|
||||||
|
connection_params,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -301,6 +304,13 @@ def run_restore(
|
||||||
restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names)
|
restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names)
|
||||||
found_names = set()
|
found_names = set()
|
||||||
remaining_restore_names = {}
|
remaining_restore_names = {}
|
||||||
|
connection_params = {
|
||||||
|
'hostname': restore_arguments.hostname,
|
||||||
|
'port': restore_arguments.port,
|
||||||
|
'username': restore_arguments.username,
|
||||||
|
'password': restore_arguments.password,
|
||||||
|
'restore_path': restore_arguments.restore_path,
|
||||||
|
}
|
||||||
|
|
||||||
for hook_name, database_names in restore_names.items():
|
for hook_name, database_names in restore_names.items():
|
||||||
for database_name in database_names:
|
for database_name in database_names:
|
||||||
|
@ -327,6 +337,7 @@ def run_restore(
|
||||||
archive_name,
|
archive_name,
|
||||||
found_hook_name or hook_name,
|
found_hook_name or hook_name,
|
||||||
dict(found_database, **{'schemas': restore_arguments.schemas}),
|
dict(found_database, **{'schemas': restore_arguments.schemas}),
|
||||||
|
connection_params,
|
||||||
)
|
)
|
||||||
|
|
||||||
# For any database that weren't found via exact matches in the hooks configuration, try to
|
# For any database that weren't found via exact matches in the hooks configuration, try to
|
||||||
|
@ -356,6 +367,7 @@ def run_restore(
|
||||||
archive_name,
|
archive_name,
|
||||||
found_hook_name or hook_name,
|
found_hook_name or hook_name,
|
||||||
dict(database, **{'schemas': restore_arguments.schemas}),
|
dict(database, **{'schemas': restore_arguments.schemas}),
|
||||||
|
connection_params,
|
||||||
)
|
)
|
||||||
|
|
||||||
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
||||||
|
|
|
@ -931,6 +931,26 @@ def make_parsers():
|
||||||
dest='schemas',
|
dest='schemas',
|
||||||
help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases',
|
help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases',
|
||||||
)
|
)
|
||||||
|
restore_group.add_argument(
|
||||||
|
'--hostname',
|
||||||
|
help='Database hostname to restore to. Defaults to the "restore_hostname" option in borgmatic\'s configuration',
|
||||||
|
)
|
||||||
|
restore_group.add_argument(
|
||||||
|
'--port',
|
||||||
|
help='Port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
|
||||||
|
)
|
||||||
|
restore_group.add_argument(
|
||||||
|
'--username',
|
||||||
|
help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration',
|
||||||
|
)
|
||||||
|
restore_group.add_argument(
|
||||||
|
'--password',
|
||||||
|
help='Password with which to connect to the restore database. Defaults to the "restore_password" option in borgmatic\'s configuration',
|
||||||
|
)
|
||||||
|
restore_group.add_argument(
|
||||||
|
'--restore-path',
|
||||||
|
help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
|
||||||
|
)
|
||||||
restore_group.add_argument(
|
restore_group.add_argument(
|
||||||
'-h', '--help', action='help', help='Show this help message and exit'
|
'-h', '--help', action='help', help='Show this help message and exit'
|
||||||
)
|
)
|
||||||
|
|
|
@ -763,10 +763,21 @@ properties:
|
||||||
Database hostname to connect to. Defaults to
|
Database hostname to connect to. Defaults to
|
||||||
connecting via local Unix socket.
|
connecting via local Unix socket.
|
||||||
example: database.example.org
|
example: database.example.org
|
||||||
|
restore_hostname:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Database hostname to restore to. Defaults to
|
||||||
|
the "hostname" option.
|
||||||
|
example: database.example.org
|
||||||
port:
|
port:
|
||||||
type: integer
|
type: integer
|
||||||
description: Port to connect to. Defaults to 5432.
|
description: Port to connect to. Defaults to 5432.
|
||||||
example: 5433
|
example: 5433
|
||||||
|
restore_port:
|
||||||
|
type: integer
|
||||||
|
description: Port to restore to. Defaults to the
|
||||||
|
"port" option.
|
||||||
|
example: 5433
|
||||||
username:
|
username:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
|
@ -775,6 +786,12 @@ properties:
|
||||||
You probably want to specify the "postgres"
|
You probably want to specify the "postgres"
|
||||||
superuser here when the database name is "all".
|
superuser here when the database name is "all".
|
||||||
example: dbuser
|
example: dbuser
|
||||||
|
restore_username:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Username with which to restore the database.
|
||||||
|
Defaults to the "username" option.
|
||||||
|
example: dbuser
|
||||||
password:
|
password:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
|
@ -784,6 +801,24 @@ properties:
|
||||||
without a password or you create a ~/.pgpass
|
without a password or you create a ~/.pgpass
|
||||||
file.
|
file.
|
||||||
example: trustsome1
|
example: trustsome1
|
||||||
|
restore_password:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Password with which to connect to the restore
|
||||||
|
database. Defaults to the "password" option.
|
||||||
|
example: trustsome1
|
||||||
|
no_owner:
|
||||||
|
type: boolean
|
||||||
|
description: |
|
||||||
|
Do not output commands to set ownership of
|
||||||
|
objects to match the original database. By
|
||||||
|
default, pg_dump and pg_restore issue ALTER
|
||||||
|
OWNER or SET SESSION AUTHORIZATION statements
|
||||||
|
to set ownership of created schema elements.
|
||||||
|
These statements will fail unless the initial
|
||||||
|
connection to the database is made by a
|
||||||
|
superuser.
|
||||||
|
example: true
|
||||||
format:
|
format:
|
||||||
type: string
|
type: string
|
||||||
enum: ['plain', 'custom', 'directory', 'tar']
|
enum: ['plain', 'custom', 'directory', 'tar']
|
||||||
|
@ -919,16 +954,33 @@ properties:
|
||||||
Database hostname to connect to. Defaults to
|
Database hostname to connect to. Defaults to
|
||||||
connecting via local Unix socket.
|
connecting via local Unix socket.
|
||||||
example: database.example.org
|
example: database.example.org
|
||||||
|
restore_hostname:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Database hostname to restore to. Defaults to
|
||||||
|
the "hostname" option.
|
||||||
|
example: database.example.org
|
||||||
port:
|
port:
|
||||||
type: integer
|
type: integer
|
||||||
description: Port to connect to. Defaults to 3306.
|
description: Port to connect to. Defaults to 3306.
|
||||||
example: 3307
|
example: 3307
|
||||||
|
restore_port:
|
||||||
|
type: integer
|
||||||
|
description: Port to restore to. Defaults to the
|
||||||
|
"port" option.
|
||||||
|
example: 5433
|
||||||
username:
|
username:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
Username with which to connect to the database.
|
Username with which to connect to the database.
|
||||||
Defaults to the username of the current user.
|
Defaults to the username of the current user.
|
||||||
example: dbuser
|
example: dbuser
|
||||||
|
restore_username:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Username with which to restore the database.
|
||||||
|
Defaults to the "username" option.
|
||||||
|
example: dbuser
|
||||||
password:
|
password:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
|
@ -937,6 +989,12 @@ properties:
|
||||||
configured to trust the configured username
|
configured to trust the configured username
|
||||||
without a password.
|
without a password.
|
||||||
example: trustsome1
|
example: trustsome1
|
||||||
|
restore_password:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Password with which to connect to the restore
|
||||||
|
database. Defaults to the "password" option.
|
||||||
|
example: trustsome1
|
||||||
format:
|
format:
|
||||||
type: string
|
type: string
|
||||||
enum: ['sql']
|
enum: ['sql']
|
||||||
|
@ -1014,6 +1072,12 @@ properties:
|
||||||
read_special and one_file_system (see above) to
|
read_special and one_file_system (see above) to
|
||||||
support dump and restore streaming.
|
support dump and restore streaming.
|
||||||
example: /var/lib/sqlite/users.db
|
example: /var/lib/sqlite/users.db
|
||||||
|
restore_path:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Path to the SQLite database file to restore to.
|
||||||
|
Defaults to the "path" option.
|
||||||
|
example: /var/lib/sqlite/users.db
|
||||||
mongodb_databases:
|
mongodb_databases:
|
||||||
type: array
|
type: array
|
||||||
items:
|
items:
|
||||||
|
@ -1036,22 +1100,45 @@ properties:
|
||||||
Database hostname to connect to. Defaults to
|
Database hostname to connect to. Defaults to
|
||||||
connecting to localhost.
|
connecting to localhost.
|
||||||
example: database.example.org
|
example: database.example.org
|
||||||
|
restore_hostname:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Database hostname to restore to. Defaults to
|
||||||
|
the "hostname" option.
|
||||||
|
example: database.example.org
|
||||||
port:
|
port:
|
||||||
type: integer
|
type: integer
|
||||||
description: Port to connect to. Defaults to 27017.
|
description: Port to connect to. Defaults to 27017.
|
||||||
example: 27018
|
example: 27018
|
||||||
|
restore_port:
|
||||||
|
type: integer
|
||||||
|
description: Port to restore to. Defaults to the
|
||||||
|
"port" option.
|
||||||
|
example: 5433
|
||||||
username:
|
username:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
Username with which to connect to the database.
|
Username with which to connect to the database.
|
||||||
Skip it if no authentication is needed.
|
Skip it if no authentication is needed.
|
||||||
example: dbuser
|
example: dbuser
|
||||||
|
restore_username:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Username with which to restore the database.
|
||||||
|
Defaults to the "username" option.
|
||||||
|
example: dbuser
|
||||||
password:
|
password:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
Password with which to connect to the database.
|
Password with which to connect to the database.
|
||||||
Skip it if no authentication is needed.
|
Skip it if no authentication is needed.
|
||||||
example: trustsome1
|
example: trustsome1
|
||||||
|
restore_password:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Password with which to connect to the restore
|
||||||
|
database. Defaults to the "password" option.
|
||||||
|
example: trustsome1
|
||||||
authentication_database:
|
authentication_database:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
|
|
|
@ -102,7 +102,9 @@ def make_database_dump_pattern(
|
||||||
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
|
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
|
||||||
|
|
||||||
|
|
||||||
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
|
def restore_database_dump(
|
||||||
|
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
|
||||||
|
):
|
||||||
'''
|
'''
|
||||||
Restore the given MongoDB database from an extract stream. The database is supplied as a
|
Restore the given MongoDB database from an extract stream. The database is supplied as a
|
||||||
one-element sequence containing a dict describing the database, as per the configuration schema.
|
one-element sequence containing a dict describing the database, as per the configuration schema.
|
||||||
|
@ -122,7 +124,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
dump_filename = dump.make_database_dump_filename(
|
dump_filename = dump.make_database_dump_filename(
|
||||||
make_dump_path(location_config), database['name'], database.get('hostname')
|
make_dump_path(location_config), database['name'], database.get('hostname')
|
||||||
)
|
)
|
||||||
restore_command = build_restore_command(extract_process, database, dump_filename)
|
restore_command = build_restore_command(
|
||||||
|
extract_process, database, dump_filename, connection_params
|
||||||
|
)
|
||||||
|
|
||||||
logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}")
|
logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}")
|
||||||
if dry_run:
|
if dry_run:
|
||||||
|
@ -138,10 +142,21 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def build_restore_command(extract_process, database, dump_filename):
|
def build_restore_command(extract_process, database, dump_filename, connection_params):
|
||||||
'''
|
'''
|
||||||
Return the mongorestore command from a single database configuration.
|
Return the mongorestore command from a single database configuration.
|
||||||
'''
|
'''
|
||||||
|
hostname = connection_params['hostname'] or database.get(
|
||||||
|
'restore_hostname', database.get('hostname')
|
||||||
|
)
|
||||||
|
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
|
||||||
|
username = connection_params['username'] or database.get(
|
||||||
|
'restore_username', database.get('username')
|
||||||
|
)
|
||||||
|
password = connection_params['password'] or database.get(
|
||||||
|
'restore_password', database.get('password')
|
||||||
|
)
|
||||||
|
|
||||||
command = ['mongorestore']
|
command = ['mongorestore']
|
||||||
if extract_process:
|
if extract_process:
|
||||||
command.append('--archive')
|
command.append('--archive')
|
||||||
|
@ -149,14 +164,14 @@ def build_restore_command(extract_process, database, dump_filename):
|
||||||
command.extend(('--dir', dump_filename))
|
command.extend(('--dir', dump_filename))
|
||||||
if database['name'] != 'all':
|
if database['name'] != 'all':
|
||||||
command.extend(('--drop', '--db', database['name']))
|
command.extend(('--drop', '--db', database['name']))
|
||||||
if 'hostname' in database:
|
if hostname:
|
||||||
command.extend(('--host', database['hostname']))
|
command.extend(('--host', hostname))
|
||||||
if 'port' in database:
|
if port:
|
||||||
command.extend(('--port', str(database['port'])))
|
command.extend(('--port', str(port)))
|
||||||
if 'username' in database:
|
if username:
|
||||||
command.extend(('--username', database['username']))
|
command.extend(('--username', username))
|
||||||
if 'password' in database:
|
if password:
|
||||||
command.extend(('--password', database['password']))
|
command.extend(('--password', password))
|
||||||
if 'authentication_database' in database:
|
if 'authentication_database' in database:
|
||||||
command.extend(('--authenticationDatabase', database['authentication_database']))
|
command.extend(('--authenticationDatabase', database['authentication_database']))
|
||||||
if 'restore_options' in database:
|
if 'restore_options' in database:
|
||||||
|
|
|
@ -185,7 +185,9 @@ def make_database_dump_pattern(
|
||||||
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
|
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
|
||||||
|
|
||||||
|
|
||||||
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
|
def restore_database_dump(
|
||||||
|
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
|
||||||
|
):
|
||||||
'''
|
'''
|
||||||
Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
|
Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
|
||||||
one-element sequence containing a dict describing the database, as per the configuration schema.
|
one-element sequence containing a dict describing the database, as per the configuration schema.
|
||||||
|
@ -199,15 +201,27 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
raise ValueError('The database configuration value is invalid')
|
raise ValueError('The database configuration value is invalid')
|
||||||
|
|
||||||
database = database_config[0]
|
database = database_config[0]
|
||||||
|
|
||||||
|
hostname = connection_params['hostname'] or database.get(
|
||||||
|
'restore_hostname', database.get('hostname')
|
||||||
|
)
|
||||||
|
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
|
||||||
|
username = connection_params['username'] or database.get(
|
||||||
|
'restore_username', database.get('username')
|
||||||
|
)
|
||||||
|
password = connection_params['password'] or database.get(
|
||||||
|
'restore_password', database.get('password')
|
||||||
|
)
|
||||||
|
|
||||||
restore_command = (
|
restore_command = (
|
||||||
('mysql', '--batch')
|
('mysql', '--batch')
|
||||||
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
|
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', hostname) if hostname else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(port)) if port else ())
|
||||||
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
|
+ (('--protocol', 'tcp') if hostname or port else ())
|
||||||
+ (('--user', database['username']) if 'username' in database else ())
|
+ (('--user', username) if username else ())
|
||||||
)
|
)
|
||||||
extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
|
extra_environment = {'MYSQL_PWD': password} if password else None
|
||||||
|
|
||||||
logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}")
|
logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}")
|
||||||
if dry_run:
|
if dry_run:
|
||||||
|
|
|
@ -23,13 +23,23 @@ def make_dump_path(location_config): # pragma: no cover
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def make_extra_environment(database):
|
def make_extra_environment(database, restore_connection_params=None):
|
||||||
'''
|
'''
|
||||||
Make the extra_environment dict from the given database configuration.
|
Make the extra_environment dict from the given database configuration.
|
||||||
|
If restore connection params are given, this is for a restore operation.
|
||||||
'''
|
'''
|
||||||
extra = dict()
|
extra = dict()
|
||||||
if 'password' in database:
|
|
||||||
|
try:
|
||||||
|
if restore_connection_params:
|
||||||
|
extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get(
|
||||||
|
'restore_password', database['password']
|
||||||
|
)
|
||||||
|
else:
|
||||||
extra['PGPASSWORD'] = database['password']
|
extra['PGPASSWORD'] = database['password']
|
||||||
|
except (AttributeError, KeyError):
|
||||||
|
pass
|
||||||
|
|
||||||
extra['PGSSLMODE'] = database.get('ssl_mode', 'disable')
|
extra['PGSSLMODE'] = database.get('ssl_mode', 'disable')
|
||||||
if 'ssl_cert' in database:
|
if 'ssl_cert' in database:
|
||||||
extra['PGSSLCERT'] = database['ssl_cert']
|
extra['PGSSLCERT'] = database['ssl_cert']
|
||||||
|
@ -135,6 +145,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
+ (('--username', database['username']) if 'username' in database else ())
|
+ (('--username', database['username']) if 'username' in database else ())
|
||||||
|
+ (('--no-owner',) if database.get('no_owner', False) else ())
|
||||||
+ (('--format', dump_format) if dump_format else ())
|
+ (('--format', dump_format) if dump_format else ())
|
||||||
+ (('--file', dump_filename) if dump_format == 'directory' else ())
|
+ (('--file', dump_filename) if dump_format == 'directory' else ())
|
||||||
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
||||||
|
@ -192,7 +203,9 @@ def make_database_dump_pattern(
|
||||||
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
|
return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
|
||||||
|
|
||||||
|
|
||||||
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
|
def restore_database_dump(
|
||||||
|
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
|
||||||
|
):
|
||||||
'''
|
'''
|
||||||
Restore the given PostgreSQL database from an extract stream. The database is supplied as a
|
Restore the given PostgreSQL database from an extract stream. The database is supplied as a
|
||||||
one-element sequence containing a dict describing the database, as per the configuration schema.
|
one-element sequence containing a dict describing the database, as per the configuration schema.
|
||||||
|
@ -202,6 +215,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
|
|
||||||
If the extract process is None, then restore the dump from the filesystem rather than from an
|
If the extract process is None, then restore the dump from the filesystem rather than from an
|
||||||
extract stream.
|
extract stream.
|
||||||
|
|
||||||
|
Use the given connection parameters to connect to the database. The connection parameters are
|
||||||
|
hostname, port, username, and password.
|
||||||
'''
|
'''
|
||||||
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
||||||
|
|
||||||
|
@ -209,6 +225,15 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
raise ValueError('The database configuration value is invalid')
|
raise ValueError('The database configuration value is invalid')
|
||||||
|
|
||||||
database = database_config[0]
|
database = database_config[0]
|
||||||
|
|
||||||
|
hostname = connection_params['hostname'] or database.get(
|
||||||
|
'restore_hostname', database.get('hostname')
|
||||||
|
)
|
||||||
|
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
|
||||||
|
username = connection_params['username'] or database.get(
|
||||||
|
'restore_username', database.get('username')
|
||||||
|
)
|
||||||
|
|
||||||
all_databases = bool(database['name'] == 'all')
|
all_databases = bool(database['name'] == 'all')
|
||||||
dump_filename = dump.make_database_dump_filename(
|
dump_filename = dump.make_database_dump_filename(
|
||||||
make_dump_path(location_config), database['name'], database.get('hostname')
|
make_dump_path(location_config), database['name'], database.get('hostname')
|
||||||
|
@ -217,9 +242,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
analyze_command = (
|
analyze_command = (
|
||||||
tuple(psql_command)
|
tuple(psql_command)
|
||||||
+ ('--no-password', '--no-psqlrc', '--quiet')
|
+ ('--no-password', '--no-psqlrc', '--quiet')
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', hostname) if hostname else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', port) if port else ())
|
||||||
+ (('--username', database['username']) if 'username' in database else ())
|
+ (('--username', username) if username else ())
|
||||||
+ (('--dbname', database['name']) if not all_databases else ())
|
+ (('--dbname', database['name']) if not all_databases else ())
|
||||||
+ (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
|
+ (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
|
||||||
+ ('--command', 'ANALYZE')
|
+ ('--command', 'ANALYZE')
|
||||||
|
@ -231,9 +256,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
+ ('--no-password',)
|
+ ('--no-password',)
|
||||||
+ (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
|
+ (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
|
||||||
+ (('--dbname', database['name']) if not all_databases else ())
|
+ (('--dbname', database['name']) if not all_databases else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', hostname) if hostname else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', port) if port else ())
|
||||||
+ (('--username', database['username']) if 'username' in database else ())
|
+ (('--username', username) if username else ())
|
||||||
|
+ (('--no-owner',) if database.get('no_owner', False) else ())
|
||||||
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
|
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
|
||||||
+ (() if extract_process else (dump_filename,))
|
+ (() if extract_process else (dump_filename,))
|
||||||
+ tuple(
|
+ tuple(
|
||||||
|
@ -243,7 +269,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
extra_environment = make_extra_environment(database)
|
extra_environment = make_extra_environment(
|
||||||
|
database, restore_connection_params=connection_params
|
||||||
|
)
|
||||||
|
|
||||||
logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}")
|
logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}")
|
||||||
if dry_run:
|
if dry_run:
|
||||||
|
|
|
@ -85,7 +85,9 @@ def make_database_dump_pattern(
|
||||||
return dump.make_database_dump_filename(make_dump_path(location_config), name)
|
return dump.make_database_dump_filename(make_dump_path(location_config), name)
|
||||||
|
|
||||||
|
|
||||||
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
|
def restore_database_dump(
|
||||||
|
database_config, log_prefix, location_config, dry_run, extract_process, connection_params
|
||||||
|
):
|
||||||
'''
|
'''
|
||||||
Restore the given SQLite3 database from an extract stream. The database is supplied as a
|
Restore the given SQLite3 database from an extract stream. The database is supplied as a
|
||||||
one-element sequence containing a dict describing the database, as per the configuration schema.
|
one-element sequence containing a dict describing the database, as per the configuration schema.
|
||||||
|
@ -98,7 +100,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
if len(database_config) != 1:
|
if len(database_config) != 1:
|
||||||
raise ValueError('The database configuration value is invalid')
|
raise ValueError('The database configuration value is invalid')
|
||||||
|
|
||||||
database_path = database_config[0]['path']
|
database_path = connection_params['restore_path'] or database_config[0].get(
|
||||||
|
'restore_path', database_config[0].get('path')
|
||||||
|
)
|
||||||
|
|
||||||
logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
|
logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
|
||||||
if dry_run:
|
if dry_run:
|
||||||
|
|
|
@ -5,16 +5,35 @@ services:
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_PASSWORD: test
|
POSTGRES_PASSWORD: test
|
||||||
POSTGRES_DB: test
|
POSTGRES_DB: test
|
||||||
|
postgresql2:
|
||||||
|
image: docker.io/postgres:13.1-alpine
|
||||||
|
environment:
|
||||||
|
POSTGRES_PASSWORD: test2
|
||||||
|
POSTGRES_DB: test
|
||||||
|
POSTGRES_USER: postgres2
|
||||||
|
command: -p 5433
|
||||||
mysql:
|
mysql:
|
||||||
image: docker.io/mariadb:10.5
|
image: docker.io/mariadb:10.5
|
||||||
environment:
|
environment:
|
||||||
MYSQL_ROOT_PASSWORD: test
|
MYSQL_ROOT_PASSWORD: test
|
||||||
MYSQL_DATABASE: test
|
MYSQL_DATABASE: test
|
||||||
|
mysql2:
|
||||||
|
image: docker.io/mariadb:10.5
|
||||||
|
environment:
|
||||||
|
MYSQL_ROOT_PASSWORD: test2
|
||||||
|
MYSQL_DATABASE: test
|
||||||
|
command: --port=3307
|
||||||
mongodb:
|
mongodb:
|
||||||
image: docker.io/mongo:5.0.5
|
image: docker.io/mongo:5.0.5
|
||||||
environment:
|
environment:
|
||||||
MONGO_INITDB_ROOT_USERNAME: root
|
MONGO_INITDB_ROOT_USERNAME: root
|
||||||
MONGO_INITDB_ROOT_PASSWORD: test
|
MONGO_INITDB_ROOT_PASSWORD: test
|
||||||
|
mongodb2:
|
||||||
|
image: docker.io/mongo:5.0.5
|
||||||
|
environment:
|
||||||
|
MONGO_INITDB_ROOT_USERNAME: root2
|
||||||
|
MONGO_INITDB_ROOT_PASSWORD: test2
|
||||||
|
command: --port=27018
|
||||||
tests:
|
tests:
|
||||||
image: docker.io/alpine:3.13
|
image: docker.io/alpine:3.13
|
||||||
environment:
|
environment:
|
||||||
|
@ -30,5 +49,8 @@ services:
|
||||||
command: --end-to-end-only
|
command: --end-to-end-only
|
||||||
depends_on:
|
depends_on:
|
||||||
- postgresql
|
- postgresql
|
||||||
|
- postgresql2
|
||||||
- mysql
|
- mysql
|
||||||
|
- mysql2
|
||||||
- mongodb
|
- mongodb
|
||||||
|
- mongodb2
|
||||||
|
|
|
@ -82,6 +82,108 @@ hooks:
|
||||||
config_file.write(config)
|
config_file.write(config)
|
||||||
|
|
||||||
|
|
||||||
|
def write_custom_restore_configuration(
|
||||||
|
source_directory,
|
||||||
|
config_path,
|
||||||
|
repository_path,
|
||||||
|
borgmatic_source_directory,
|
||||||
|
postgresql_dump_format='custom',
|
||||||
|
mongodb_dump_format='archive',
|
||||||
|
):
|
||||||
|
'''
|
||||||
|
Write out borgmatic configuration into a file at the config path. Set the options so as to work
|
||||||
|
for testing with custom restore options. This includes a custom restore_hostname, restore_port,
|
||||||
|
restore_username, restore_password and restore_path.
|
||||||
|
'''
|
||||||
|
config = f'''
|
||||||
|
location:
|
||||||
|
source_directories:
|
||||||
|
- {source_directory}
|
||||||
|
repositories:
|
||||||
|
- {repository_path}
|
||||||
|
borgmatic_source_directory: {borgmatic_source_directory}
|
||||||
|
|
||||||
|
storage:
|
||||||
|
encryption_passphrase: "test"
|
||||||
|
|
||||||
|
hooks:
|
||||||
|
postgresql_databases:
|
||||||
|
- name: test
|
||||||
|
hostname: postgresql
|
||||||
|
username: postgres
|
||||||
|
password: test
|
||||||
|
format: {postgresql_dump_format}
|
||||||
|
restore_hostname: postgresql2
|
||||||
|
restore_port: 5433
|
||||||
|
restore_username: postgres2
|
||||||
|
restore_password: test2
|
||||||
|
mysql_databases:
|
||||||
|
- name: test
|
||||||
|
hostname: mysql
|
||||||
|
username: root
|
||||||
|
password: test
|
||||||
|
restore_hostname: mysql2
|
||||||
|
restore_port: 3307
|
||||||
|
restore_username: root
|
||||||
|
restore_password: test2
|
||||||
|
mongodb_databases:
|
||||||
|
- name: test
|
||||||
|
hostname: mongodb
|
||||||
|
username: root
|
||||||
|
password: test
|
||||||
|
authentication_database: admin
|
||||||
|
format: {mongodb_dump_format}
|
||||||
|
restore_hostname: mongodb2
|
||||||
|
restore_port: 27018
|
||||||
|
restore_username: root2
|
||||||
|
restore_password: test2
|
||||||
|
sqlite_databases:
|
||||||
|
- name: sqlite_test
|
||||||
|
path: /tmp/sqlite_test.db
|
||||||
|
restore_path: /tmp/sqlite_test2.db
|
||||||
|
'''
|
||||||
|
|
||||||
|
with open(config_path, 'w') as config_file:
|
||||||
|
config_file.write(config)
|
||||||
|
|
||||||
|
|
||||||
|
def write_simple_custom_restore_configuration(
|
||||||
|
source_directory,
|
||||||
|
config_path,
|
||||||
|
repository_path,
|
||||||
|
borgmatic_source_directory,
|
||||||
|
postgresql_dump_format='custom',
|
||||||
|
):
|
||||||
|
'''
|
||||||
|
Write out borgmatic configuration into a file at the config path. Set the options so as to work
|
||||||
|
for testing with custom restore options, but this time using CLI arguments. This includes a
|
||||||
|
custom restore_hostname, restore_port, restore_username and restore_password as we only test
|
||||||
|
these options for PostgreSQL.
|
||||||
|
'''
|
||||||
|
config = f'''
|
||||||
|
location:
|
||||||
|
source_directories:
|
||||||
|
- {source_directory}
|
||||||
|
repositories:
|
||||||
|
- {repository_path}
|
||||||
|
borgmatic_source_directory: {borgmatic_source_directory}
|
||||||
|
|
||||||
|
storage:
|
||||||
|
encryption_passphrase: "test"
|
||||||
|
|
||||||
|
hooks:
|
||||||
|
postgresql_databases:
|
||||||
|
- name: test
|
||||||
|
hostname: postgresql
|
||||||
|
username: postgres
|
||||||
|
password: test
|
||||||
|
format: {postgresql_dump_format}
|
||||||
|
'''
|
||||||
|
|
||||||
|
with open(config_path, 'w') as config_file:
|
||||||
|
config_file.write(config)
|
||||||
|
|
||||||
|
|
||||||
def test_database_dump_and_restore():
|
def test_database_dump_and_restore():
|
||||||
# Create a Borg repository.
|
# Create a Borg repository.
|
||||||
temporary_directory = tempfile.mkdtemp()
|
temporary_directory = tempfile.mkdtemp()
|
||||||
|
@ -125,6 +227,103 @@ def test_database_dump_and_restore():
|
||||||
shutil.rmtree(temporary_directory)
|
shutil.rmtree(temporary_directory)
|
||||||
|
|
||||||
|
|
||||||
|
def test_database_dump_and_restore_with_restore_cli_arguments():
|
||||||
|
# Create a Borg repository.
|
||||||
|
temporary_directory = tempfile.mkdtemp()
|
||||||
|
repository_path = os.path.join(temporary_directory, 'test.borg')
|
||||||
|
borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
|
||||||
|
|
||||||
|
original_working_directory = os.getcwd()
|
||||||
|
|
||||||
|
try:
|
||||||
|
config_path = os.path.join(temporary_directory, 'test.yaml')
|
||||||
|
write_simple_custom_restore_configuration(
|
||||||
|
temporary_directory, config_path, repository_path, borgmatic_source_directory
|
||||||
|
)
|
||||||
|
|
||||||
|
subprocess.check_call(
|
||||||
|
['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
|
||||||
|
)
|
||||||
|
|
||||||
|
# Run borgmatic to generate a backup archive including a database dump.
|
||||||
|
subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
|
||||||
|
|
||||||
|
# Get the created archive name.
|
||||||
|
output = subprocess.check_output(
|
||||||
|
['borgmatic', '--config', config_path, 'list', '--json']
|
||||||
|
).decode(sys.stdout.encoding)
|
||||||
|
parsed_output = json.loads(output)
|
||||||
|
|
||||||
|
assert len(parsed_output) == 1
|
||||||
|
assert len(parsed_output[0]['archives']) == 1
|
||||||
|
archive_name = parsed_output[0]['archives'][0]['archive']
|
||||||
|
|
||||||
|
# Restore the database from the archive.
|
||||||
|
subprocess.check_call(
|
||||||
|
[
|
||||||
|
'borgmatic',
|
||||||
|
'-v',
|
||||||
|
'2',
|
||||||
|
'--config',
|
||||||
|
config_path,
|
||||||
|
'restore',
|
||||||
|
'--archive',
|
||||||
|
archive_name,
|
||||||
|
'--hostname',
|
||||||
|
'postgresql2',
|
||||||
|
'--port',
|
||||||
|
'5433',
|
||||||
|
'--username',
|
||||||
|
'postgres2',
|
||||||
|
'--password',
|
||||||
|
'test2',
|
||||||
|
]
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
os.chdir(original_working_directory)
|
||||||
|
shutil.rmtree(temporary_directory)
|
||||||
|
|
||||||
|
|
||||||
|
def test_database_dump_and_restore_with_restore_configuration_options():
|
||||||
|
# Create a Borg repository.
|
||||||
|
temporary_directory = tempfile.mkdtemp()
|
||||||
|
repository_path = os.path.join(temporary_directory, 'test.borg')
|
||||||
|
borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
|
||||||
|
|
||||||
|
original_working_directory = os.getcwd()
|
||||||
|
|
||||||
|
try:
|
||||||
|
config_path = os.path.join(temporary_directory, 'test.yaml')
|
||||||
|
write_custom_restore_configuration(
|
||||||
|
temporary_directory, config_path, repository_path, borgmatic_source_directory
|
||||||
|
)
|
||||||
|
|
||||||
|
subprocess.check_call(
|
||||||
|
['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
|
||||||
|
)
|
||||||
|
|
||||||
|
# Run borgmatic to generate a backup archive including a database dump.
|
||||||
|
subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
|
||||||
|
|
||||||
|
# Get the created archive name.
|
||||||
|
output = subprocess.check_output(
|
||||||
|
['borgmatic', '--config', config_path, 'list', '--json']
|
||||||
|
).decode(sys.stdout.encoding)
|
||||||
|
parsed_output = json.loads(output)
|
||||||
|
|
||||||
|
assert len(parsed_output) == 1
|
||||||
|
assert len(parsed_output[0]['archives']) == 1
|
||||||
|
archive_name = parsed_output[0]['archives'][0]['archive']
|
||||||
|
|
||||||
|
# Restore the database from the archive.
|
||||||
|
subprocess.check_call(
|
||||||
|
['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name]
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
os.chdir(original_working_directory)
|
||||||
|
shutil.rmtree(temporary_directory)
|
||||||
|
|
||||||
|
|
||||||
def test_database_dump_and_restore_with_directory_format():
|
def test_database_dump_and_restore_with_directory_format():
|
||||||
# Create a Borg repository.
|
# Create a Borg repository.
|
||||||
temporary_directory = tempfile.mkdtemp()
|
temporary_directory = tempfile.mkdtemp()
|
||||||
|
|
|
@ -241,6 +241,7 @@ def test_run_restore_restores_each_database():
|
||||||
archive_name=object,
|
archive_name=object,
|
||||||
hook_name='postgresql_databases',
|
hook_name='postgresql_databases',
|
||||||
database={'name': 'foo', 'schemas': None},
|
database={'name': 'foo', 'schemas': None},
|
||||||
|
connection_params=object,
|
||||||
).once()
|
).once()
|
||||||
flexmock(module).should_receive('restore_single_database').with_args(
|
flexmock(module).should_receive('restore_single_database').with_args(
|
||||||
repository=object,
|
repository=object,
|
||||||
|
@ -254,6 +255,7 @@ def test_run_restore_restores_each_database():
|
||||||
archive_name=object,
|
archive_name=object,
|
||||||
hook_name='postgresql_databases',
|
hook_name='postgresql_databases',
|
||||||
database={'name': 'bar', 'schemas': None},
|
database={'name': 'bar', 'schemas': None},
|
||||||
|
connection_params=object,
|
||||||
).once()
|
).once()
|
||||||
flexmock(module).should_receive('ensure_databases_found')
|
flexmock(module).should_receive('ensure_databases_found')
|
||||||
|
|
||||||
|
@ -264,7 +266,15 @@ def test_run_restore_restores_each_database():
|
||||||
hooks=flexmock(),
|
hooks=flexmock(),
|
||||||
local_borg_version=flexmock(),
|
local_borg_version=flexmock(),
|
||||||
restore_arguments=flexmock(
|
restore_arguments=flexmock(
|
||||||
repository='repo', archive='archive', databases=flexmock(), schemas=None
|
repository='repo',
|
||||||
|
archive='archive',
|
||||||
|
databases=flexmock(),
|
||||||
|
schemas=None,
|
||||||
|
hostname=None,
|
||||||
|
port=None,
|
||||||
|
username=None,
|
||||||
|
password=None,
|
||||||
|
restore_path=None,
|
||||||
),
|
),
|
||||||
global_arguments=flexmock(dry_run=False),
|
global_arguments=flexmock(dry_run=False),
|
||||||
local_path=flexmock(),
|
local_path=flexmock(),
|
||||||
|
@ -337,6 +347,7 @@ def test_run_restore_restores_database_configured_with_all_name():
|
||||||
archive_name=object,
|
archive_name=object,
|
||||||
hook_name='postgresql_databases',
|
hook_name='postgresql_databases',
|
||||||
database={'name': 'foo', 'schemas': None},
|
database={'name': 'foo', 'schemas': None},
|
||||||
|
connection_params=object,
|
||||||
).once()
|
).once()
|
||||||
flexmock(module).should_receive('restore_single_database').with_args(
|
flexmock(module).should_receive('restore_single_database').with_args(
|
||||||
repository=object,
|
repository=object,
|
||||||
|
@ -350,6 +361,7 @@ def test_run_restore_restores_database_configured_with_all_name():
|
||||||
archive_name=object,
|
archive_name=object,
|
||||||
hook_name='postgresql_databases',
|
hook_name='postgresql_databases',
|
||||||
database={'name': 'bar', 'schemas': None},
|
database={'name': 'bar', 'schemas': None},
|
||||||
|
connection_params=object,
|
||||||
).once()
|
).once()
|
||||||
flexmock(module).should_receive('ensure_databases_found')
|
flexmock(module).should_receive('ensure_databases_found')
|
||||||
|
|
||||||
|
@ -360,7 +372,15 @@ def test_run_restore_restores_database_configured_with_all_name():
|
||||||
hooks=flexmock(),
|
hooks=flexmock(),
|
||||||
local_borg_version=flexmock(),
|
local_borg_version=flexmock(),
|
||||||
restore_arguments=flexmock(
|
restore_arguments=flexmock(
|
||||||
repository='repo', archive='archive', databases=flexmock(), schemas=None
|
repository='repo',
|
||||||
|
archive='archive',
|
||||||
|
databases=flexmock(),
|
||||||
|
schemas=None,
|
||||||
|
hostname=None,
|
||||||
|
port=None,
|
||||||
|
username=None,
|
||||||
|
password=None,
|
||||||
|
restore_path=None,
|
||||||
),
|
),
|
||||||
global_arguments=flexmock(dry_run=False),
|
global_arguments=flexmock(dry_run=False),
|
||||||
local_path=flexmock(),
|
local_path=flexmock(),
|
||||||
|
@ -411,6 +431,7 @@ def test_run_restore_skips_missing_database():
|
||||||
archive_name=object,
|
archive_name=object,
|
||||||
hook_name='postgresql_databases',
|
hook_name='postgresql_databases',
|
||||||
database={'name': 'foo', 'schemas': None},
|
database={'name': 'foo', 'schemas': None},
|
||||||
|
connection_params=object,
|
||||||
).once()
|
).once()
|
||||||
flexmock(module).should_receive('restore_single_database').with_args(
|
flexmock(module).should_receive('restore_single_database').with_args(
|
||||||
repository=object,
|
repository=object,
|
||||||
|
@ -424,6 +445,7 @@ def test_run_restore_skips_missing_database():
|
||||||
archive_name=object,
|
archive_name=object,
|
||||||
hook_name='postgresql_databases',
|
hook_name='postgresql_databases',
|
||||||
database={'name': 'bar', 'schemas': None},
|
database={'name': 'bar', 'schemas': None},
|
||||||
|
connection_params=object,
|
||||||
).never()
|
).never()
|
||||||
flexmock(module).should_receive('ensure_databases_found')
|
flexmock(module).should_receive('ensure_databases_found')
|
||||||
|
|
||||||
|
@ -434,7 +456,15 @@ def test_run_restore_skips_missing_database():
|
||||||
hooks=flexmock(),
|
hooks=flexmock(),
|
||||||
local_borg_version=flexmock(),
|
local_borg_version=flexmock(),
|
||||||
restore_arguments=flexmock(
|
restore_arguments=flexmock(
|
||||||
repository='repo', archive='archive', databases=flexmock(), schemas=None
|
repository='repo',
|
||||||
|
archive='archive',
|
||||||
|
databases=flexmock(),
|
||||||
|
schemas=None,
|
||||||
|
hostname=None,
|
||||||
|
port=None,
|
||||||
|
username=None,
|
||||||
|
password=None,
|
||||||
|
restore_path=None,
|
||||||
),
|
),
|
||||||
global_arguments=flexmock(dry_run=False),
|
global_arguments=flexmock(dry_run=False),
|
||||||
local_path=flexmock(),
|
local_path=flexmock(),
|
||||||
|
@ -479,6 +509,7 @@ def test_run_restore_restores_databases_from_different_hooks():
|
||||||
archive_name=object,
|
archive_name=object,
|
||||||
hook_name='postgresql_databases',
|
hook_name='postgresql_databases',
|
||||||
database={'name': 'foo', 'schemas': None},
|
database={'name': 'foo', 'schemas': None},
|
||||||
|
connection_params=object,
|
||||||
).once()
|
).once()
|
||||||
flexmock(module).should_receive('restore_single_database').with_args(
|
flexmock(module).should_receive('restore_single_database').with_args(
|
||||||
repository=object,
|
repository=object,
|
||||||
|
@ -492,6 +523,7 @@ def test_run_restore_restores_databases_from_different_hooks():
|
||||||
archive_name=object,
|
archive_name=object,
|
||||||
hook_name='mysql_databases',
|
hook_name='mysql_databases',
|
||||||
database={'name': 'bar', 'schemas': None},
|
database={'name': 'bar', 'schemas': None},
|
||||||
|
connection_params=object,
|
||||||
).once()
|
).once()
|
||||||
flexmock(module).should_receive('ensure_databases_found')
|
flexmock(module).should_receive('ensure_databases_found')
|
||||||
|
|
||||||
|
@ -502,7 +534,15 @@ def test_run_restore_restores_databases_from_different_hooks():
|
||||||
hooks=flexmock(),
|
hooks=flexmock(),
|
||||||
local_borg_version=flexmock(),
|
local_borg_version=flexmock(),
|
||||||
restore_arguments=flexmock(
|
restore_arguments=flexmock(
|
||||||
repository='repo', archive='archive', databases=flexmock(), schemas=None
|
repository='repo',
|
||||||
|
archive='archive',
|
||||||
|
databases=flexmock(),
|
||||||
|
schemas=None,
|
||||||
|
hostname=None,
|
||||||
|
port=None,
|
||||||
|
username=None,
|
||||||
|
password=None,
|
||||||
|
restore_path=None,
|
||||||
),
|
),
|
||||||
global_arguments=flexmock(dry_run=False),
|
global_arguments=flexmock(dry_run=False),
|
||||||
local_path=flexmock(),
|
local_path=flexmock(),
|
||||||
|
|
|
@ -171,7 +171,17 @@ def test_restore_database_dump_runs_mongorestore():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -185,7 +195,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=flexmock(),
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -215,7 +235,17 @@ def test_restore_database_dump_runs_mongorestore_with_hostname_and_port():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -253,7 +283,129 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
|
||||||
|
database_config = [
|
||||||
|
{
|
||||||
|
'name': 'foo',
|
||||||
|
'username': 'mongo',
|
||||||
|
'password': 'trustsome1',
|
||||||
|
'authentication_database': 'admin',
|
||||||
|
'restore_hostname': 'restorehost',
|
||||||
|
'restore_port': 'restoreport',
|
||||||
|
'restore_username': 'restoreusername',
|
||||||
|
'restore_password': 'restorepassword',
|
||||||
|
'schemas': None,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('make_dump_path')
|
||||||
|
flexmock(module.dump).should_receive('make_database_dump_filename')
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
[
|
||||||
|
'mongorestore',
|
||||||
|
'--archive',
|
||||||
|
'--drop',
|
||||||
|
'--db',
|
||||||
|
'foo',
|
||||||
|
'--host',
|
||||||
|
'clihost',
|
||||||
|
'--port',
|
||||||
|
'cliport',
|
||||||
|
'--username',
|
||||||
|
'cliusername',
|
||||||
|
'--password',
|
||||||
|
'clipassword',
|
||||||
|
'--authenticationDatabase',
|
||||||
|
'admin',
|
||||||
|
],
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
).once()
|
||||||
|
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': 'clihost',
|
||||||
|
'port': 'cliport',
|
||||||
|
'username': 'cliusername',
|
||||||
|
'password': 'clipassword',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
|
||||||
|
database_config = [
|
||||||
|
{
|
||||||
|
'name': 'foo',
|
||||||
|
'username': 'mongo',
|
||||||
|
'password': 'trustsome1',
|
||||||
|
'authentication_database': 'admin',
|
||||||
|
'schemas': None,
|
||||||
|
'restore_hostname': 'restorehost',
|
||||||
|
'restore_port': 'restoreport',
|
||||||
|
'restore_username': 'restoreuser',
|
||||||
|
'restore_password': 'restorepass',
|
||||||
|
}
|
||||||
|
]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('make_dump_path')
|
||||||
|
flexmock(module.dump).should_receive('make_database_dump_filename')
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
[
|
||||||
|
'mongorestore',
|
||||||
|
'--archive',
|
||||||
|
'--drop',
|
||||||
|
'--db',
|
||||||
|
'foo',
|
||||||
|
'--host',
|
||||||
|
'restorehost',
|
||||||
|
'--port',
|
||||||
|
'restoreport',
|
||||||
|
'--username',
|
||||||
|
'restoreuser',
|
||||||
|
'--password',
|
||||||
|
'restorepass',
|
||||||
|
'--authenticationDatabase',
|
||||||
|
'admin',
|
||||||
|
],
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
).once()
|
||||||
|
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -271,7 +423,17 @@ def test_restore_database_dump_runs_mongorestore_with_options():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -299,7 +461,17 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -317,7 +489,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -329,7 +511,17 @@ def test_restore_database_dump_with_dry_run_skips_restore():
|
||||||
flexmock(module).should_receive('execute_command_with_processes').never()
|
flexmock(module).should_receive('execute_command_with_processes').never()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=True,
|
||||||
|
extract_process=flexmock(),
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -346,5 +538,15 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=None
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=None,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
|
@ -392,7 +392,17 @@ def test_restore_database_dump_runs_mysql_to_restore():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -404,7 +414,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=flexmock(),
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -421,7 +441,17 @@ def test_restore_database_dump_runs_mysql_with_options():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -447,7 +477,17 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -464,7 +504,115 @@ def test_restore_database_dump_runs_mysql_with_username_and_password():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
|
||||||
|
database_config = [
|
||||||
|
{
|
||||||
|
'name': 'foo',
|
||||||
|
'username': 'root',
|
||||||
|
'password': 'trustsome1',
|
||||||
|
'restore_hostname': 'restorehost',
|
||||||
|
'restore_port': 'restoreport',
|
||||||
|
'restore_username': 'restoreusername',
|
||||||
|
'restore_password': 'restorepassword',
|
||||||
|
}
|
||||||
|
]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
(
|
||||||
|
'mysql',
|
||||||
|
'--batch',
|
||||||
|
'--host',
|
||||||
|
'clihost',
|
||||||
|
'--port',
|
||||||
|
'cliport',
|
||||||
|
'--protocol',
|
||||||
|
'tcp',
|
||||||
|
'--user',
|
||||||
|
'cliusername',
|
||||||
|
),
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment={'MYSQL_PWD': 'clipassword'},
|
||||||
|
).once()
|
||||||
|
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': 'clihost',
|
||||||
|
'port': 'cliport',
|
||||||
|
'username': 'cliusername',
|
||||||
|
'password': 'clipassword',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
|
||||||
|
database_config = [
|
||||||
|
{
|
||||||
|
'name': 'foo',
|
||||||
|
'username': 'root',
|
||||||
|
'password': 'trustsome1',
|
||||||
|
'hostname': 'dbhost',
|
||||||
|
'port': 'dbport',
|
||||||
|
'restore_username': 'restoreuser',
|
||||||
|
'restore_password': 'restorepass',
|
||||||
|
'restore_hostname': 'restorehost',
|
||||||
|
'restore_port': 'restoreport',
|
||||||
|
}
|
||||||
|
]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
(
|
||||||
|
'mysql',
|
||||||
|
'--batch',
|
||||||
|
'--host',
|
||||||
|
'restorehost',
|
||||||
|
'--port',
|
||||||
|
'restoreport',
|
||||||
|
'--protocol',
|
||||||
|
'tcp',
|
||||||
|
'--user',
|
||||||
|
'restoreuser',
|
||||||
|
),
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment={'MYSQL_PWD': 'restorepass'},
|
||||||
|
).once()
|
||||||
|
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -474,5 +622,15 @@ def test_restore_database_dump_with_dry_run_skips_restore():
|
||||||
flexmock(module).should_receive('execute_command_with_processes').never()
|
flexmock(module).should_receive('execute_command_with_processes').never()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=True,
|
||||||
|
extract_process=flexmock(),
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
|
@ -479,7 +479,17 @@ def test_restore_database_dump_runs_pg_restore():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -494,7 +504,17 @@ def test_restore_database_dump_errors_on_multiple_database_config():
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=flexmock(),
|
||||||
|
connection_params={
|
||||||
|
'restore_hostname': None,
|
||||||
|
'restore_port': None,
|
||||||
|
'restore_username': None,
|
||||||
|
'restore_password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -545,7 +565,17 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -594,7 +624,183 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_extra_environment_with_cli_password_sets_correct_password():
|
||||||
|
database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'}
|
||||||
|
|
||||||
|
extra = module.make_extra_environment(
|
||||||
|
database, restore_connection_params={'password': 'clipassword'}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert extra['PGPASSWORD'] == 'clipassword'
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
|
||||||
|
database_config = [
|
||||||
|
{
|
||||||
|
'name': 'foo',
|
||||||
|
'hostname': 'database.example.org',
|
||||||
|
'port': 5433,
|
||||||
|
'username': 'postgres',
|
||||||
|
'password': 'trustsome1',
|
||||||
|
'restore_hostname': 'restorehost',
|
||||||
|
'restore_port': 'restoreport',
|
||||||
|
'restore_username': 'restoreusername',
|
||||||
|
'restore_password': 'restorepassword',
|
||||||
|
'schemas': None,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('make_extra_environment').and_return(
|
||||||
|
{'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}
|
||||||
|
)
|
||||||
|
flexmock(module).should_receive('make_dump_path')
|
||||||
|
flexmock(module.dump).should_receive('make_database_dump_filename')
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
(
|
||||||
|
'pg_restore',
|
||||||
|
'--no-password',
|
||||||
|
'--if-exists',
|
||||||
|
'--exit-on-error',
|
||||||
|
'--clean',
|
||||||
|
'--dbname',
|
||||||
|
'foo',
|
||||||
|
'--host',
|
||||||
|
'clihost',
|
||||||
|
'--port',
|
||||||
|
'cliport',
|
||||||
|
'--username',
|
||||||
|
'cliusername',
|
||||||
|
),
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
|
||||||
|
).once()
|
||||||
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
|
(
|
||||||
|
'psql',
|
||||||
|
'--no-password',
|
||||||
|
'--no-psqlrc',
|
||||||
|
'--quiet',
|
||||||
|
'--host',
|
||||||
|
'clihost',
|
||||||
|
'--port',
|
||||||
|
'cliport',
|
||||||
|
'--username',
|
||||||
|
'cliusername',
|
||||||
|
'--dbname',
|
||||||
|
'foo',
|
||||||
|
'--command',
|
||||||
|
'ANALYZE',
|
||||||
|
),
|
||||||
|
extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
|
||||||
|
).once()
|
||||||
|
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': 'clihost',
|
||||||
|
'port': 'cliport',
|
||||||
|
'username': 'cliusername',
|
||||||
|
'password': 'clipassword',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
|
||||||
|
database_config = [
|
||||||
|
{
|
||||||
|
'name': 'foo',
|
||||||
|
'hostname': 'database.example.org',
|
||||||
|
'port': 5433,
|
||||||
|
'username': 'postgres',
|
||||||
|
'password': 'trustsome1',
|
||||||
|
'schemas': None,
|
||||||
|
'restore_hostname': 'restorehost',
|
||||||
|
'restore_port': 'restoreport',
|
||||||
|
'restore_username': 'restoreusername',
|
||||||
|
'restore_password': 'restorepassword',
|
||||||
|
}
|
||||||
|
]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('make_extra_environment').and_return(
|
||||||
|
{'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}
|
||||||
|
)
|
||||||
|
flexmock(module).should_receive('make_dump_path')
|
||||||
|
flexmock(module.dump).should_receive('make_database_dump_filename')
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
(
|
||||||
|
'pg_restore',
|
||||||
|
'--no-password',
|
||||||
|
'--if-exists',
|
||||||
|
'--exit-on-error',
|
||||||
|
'--clean',
|
||||||
|
'--dbname',
|
||||||
|
'foo',
|
||||||
|
'--host',
|
||||||
|
'restorehost',
|
||||||
|
'--port',
|
||||||
|
'restoreport',
|
||||||
|
'--username',
|
||||||
|
'restoreusername',
|
||||||
|
),
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
|
||||||
|
).once()
|
||||||
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
|
(
|
||||||
|
'psql',
|
||||||
|
'--no-password',
|
||||||
|
'--no-psqlrc',
|
||||||
|
'--quiet',
|
||||||
|
'--host',
|
||||||
|
'restorehost',
|
||||||
|
'--port',
|
||||||
|
'restoreport',
|
||||||
|
'--username',
|
||||||
|
'restoreusername',
|
||||||
|
'--dbname',
|
||||||
|
'foo',
|
||||||
|
'--command',
|
||||||
|
'ANALYZE',
|
||||||
|
),
|
||||||
|
extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
|
||||||
|
).once()
|
||||||
|
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -644,7 +850,17 @@ def test_restore_database_dump_runs_pg_restore_with_options():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -672,7 +888,17 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -705,7 +931,17 @@ def test_restore_database_dump_runs_psql_for_plain_database_dump():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -759,7 +995,17 @@ def test_restore_database_dump_runs_non_default_pg_restore_and_psql():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -772,7 +1018,17 @@ def test_restore_database_dump_with_dry_run_skips_restore():
|
||||||
flexmock(module).should_receive('execute_command_with_processes').never()
|
flexmock(module).should_receive('execute_command_with_processes').never()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=True,
|
||||||
|
extract_process=flexmock(),
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -813,7 +1069,17 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=None
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=None,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -858,5 +1124,15 @@ def test_restore_database_dump_with_schemas_restores_schemas():
|
||||||
).once()
|
).once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=None
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=None,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import logging
|
||||||
import pytest
|
import pytest
|
||||||
from flexmock import flexmock
|
from flexmock import flexmock
|
||||||
|
|
||||||
|
@ -94,12 +95,81 @@ def test_restore_database_dump_restores_database():
|
||||||
database_config = [{'path': '/path/to/database', 'name': 'database'}]
|
database_config = [{'path': '/path/to/database', 'name': 'database'}]
|
||||||
extract_process = flexmock(stdout=flexmock())
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
flexmock(module).should_receive('execute_command_with_processes').once()
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
(
|
||||||
|
'sqlite3',
|
||||||
|
'/path/to/database',
|
||||||
|
),
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
).once()
|
||||||
|
|
||||||
flexmock(module.os).should_receive('remove').once()
|
flexmock(module.os).should_receive('remove').once()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={'restore_path': None},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_with_connection_params_uses_connection_params_for_restore():
|
||||||
|
database_config = [
|
||||||
|
{'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
|
||||||
|
]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
(
|
||||||
|
'sqlite3',
|
||||||
|
'cli/path/to/database',
|
||||||
|
),
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
).once()
|
||||||
|
|
||||||
|
flexmock(module.os).should_receive('remove').once()
|
||||||
|
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={'restore_path': 'cli/path/to/database'},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_without_connection_params_uses_restore_params_in_config_for_restore():
|
||||||
|
database_config = [
|
||||||
|
{'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'}
|
||||||
|
]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
(
|
||||||
|
'sqlite3',
|
||||||
|
'config/path/to/database',
|
||||||
|
),
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
).once()
|
||||||
|
|
||||||
|
flexmock(module.os).should_receive('remove').once()
|
||||||
|
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={'restore_path': None},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -111,7 +181,12 @@ def test_restore_database_dump_does_not_restore_database_if_dry_run():
|
||||||
flexmock(module.os).should_receive('remove').never()
|
flexmock(module.os).should_receive('remove').never()
|
||||||
|
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=True, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=True,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={'restore_path': None},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -121,5 +196,10 @@ def test_restore_database_dump_raises_error_if_database_config_is_invalid():
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
module.restore_database_dump(
|
module.restore_database_dump(
|
||||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
database_config,
|
||||||
|
'test.yaml',
|
||||||
|
{},
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={'restore_path': None},
|
||||||
)
|
)
|
||||||
|
|
Loading…
Reference in a new issue