Custom dump command options for MySQL and MariaDB.
Merge pull request #81 from shivansh02/feature/custom-dump-restore-commands-mysql
This commit is contained in:
commit
a1153a21fa
5 changed files with 234 additions and 6 deletions
|
@ -971,6 +971,22 @@ properties:
|
||||||
a password will only work if MariaDB is configured to
|
a password will only work if MariaDB is configured to
|
||||||
trust the configured username without a password.
|
trust the configured username without a password.
|
||||||
example: trustsome1
|
example: trustsome1
|
||||||
|
mariadb_dump_command:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Command to use instead of "mariadb-dump". This can
|
||||||
|
be used to run a specific mariadb_dump version
|
||||||
|
(e.g., one inside a running container).
|
||||||
|
Defaults to "mariadb-dump".
|
||||||
|
example: docker exec mariadb_container mariadb-dump
|
||||||
|
mariadb_command:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Command to run instead of "mariadb". This
|
||||||
|
can be used to run a specific mariadb
|
||||||
|
version (e.g., one inside a running container).
|
||||||
|
Defaults to "mariadb".
|
||||||
|
example: docker exec mariadb_container mariadb
|
||||||
restore_password:
|
restore_password:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
|
@ -1087,6 +1103,21 @@ properties:
|
||||||
Password with which to connect to the restore database.
|
Password with which to connect to the restore database.
|
||||||
Defaults to the "password" option.
|
Defaults to the "password" option.
|
||||||
example: trustsome1
|
example: trustsome1
|
||||||
|
mysql_dump_command:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Command to use instead of "mysqldump". This can be used
|
||||||
|
to run a specific mysql_dump version (e.g., one inside
|
||||||
|
a running container). Defaults to "mysqldump".
|
||||||
|
example: docker exec mysql_container mysqldump
|
||||||
|
mysql_command:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Command to run instead of "mysql". This
|
||||||
|
can be used to run a specific mysql
|
||||||
|
version (e.g., one inside a running container).
|
||||||
|
Defaults to "mysql".
|
||||||
|
example: docker exec mysql_container mysql
|
||||||
format:
|
format:
|
||||||
type: string
|
type: string
|
||||||
enum: ['sql']
|
enum: ['sql']
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
import copy
|
import copy
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import shlex
|
||||||
|
|
||||||
from borgmatic.execute import (
|
from borgmatic.execute import (
|
||||||
execute_command,
|
execute_command,
|
||||||
|
@ -35,8 +36,11 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
|
||||||
if dry_run:
|
if dry_run:
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
|
mariadb_show_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb')
|
||||||
|
)
|
||||||
show_command = (
|
show_command = (
|
||||||
('mariadb',)
|
mariadb_show_command
|
||||||
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
|
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
|
@ -79,8 +83,12 @@ def execute_dump_command(
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
mariadb_dump_command = tuple(
|
||||||
|
shlex.quote(part)
|
||||||
|
for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump')
|
||||||
|
)
|
||||||
dump_command = (
|
dump_command = (
|
||||||
('mariadb-dump',)
|
mariadb_dump_command
|
||||||
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
||||||
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
|
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
|
@ -208,8 +216,12 @@ def restore_data_source_dump(
|
||||||
'restore_password', data_source.get('password')
|
'restore_password', data_source.get('password')
|
||||||
)
|
)
|
||||||
|
|
||||||
|
mariadb_restore_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb')
|
||||||
|
)
|
||||||
restore_command = (
|
restore_command = (
|
||||||
('mariadb', '--batch')
|
mariadb_restore_command
|
||||||
|
+ ('--batch',)
|
||||||
+ (
|
+ (
|
||||||
tuple(data_source['restore_options'].split(' '))
|
tuple(data_source['restore_options'].split(' '))
|
||||||
if 'restore_options' in data_source
|
if 'restore_options' in data_source
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
import copy
|
import copy
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import shlex
|
||||||
|
|
||||||
from borgmatic.execute import (
|
from borgmatic.execute import (
|
||||||
execute_command,
|
execute_command,
|
||||||
|
@ -35,8 +36,11 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
|
||||||
if dry_run:
|
if dry_run:
|
||||||
return ()
|
return ()
|
||||||
|
|
||||||
|
mysql_show_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(database.get('mysql_command') or 'mysql')
|
||||||
|
)
|
||||||
show_command = (
|
show_command = (
|
||||||
('mysql',)
|
mysql_show_command
|
||||||
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
|
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
|
@ -79,8 +83,11 @@ def execute_dump_command(
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
mysql_dump_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(database.get('mysql_dump_command') or 'mysqldump')
|
||||||
|
)
|
||||||
dump_command = (
|
dump_command = (
|
||||||
('mysqldump',)
|
mysql_dump_command
|
||||||
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
||||||
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
|
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
|
@ -207,8 +214,12 @@ def restore_data_source_dump(
|
||||||
'restore_password', data_source.get('password')
|
'restore_password', data_source.get('password')
|
||||||
)
|
)
|
||||||
|
|
||||||
|
mysql_restore_command = tuple(
|
||||||
|
shlex.quote(part) for part in shlex.split(data_source.get('mysql_command') or 'mysql')
|
||||||
|
)
|
||||||
restore_command = (
|
restore_command = (
|
||||||
('mysql', '--batch')
|
mysql_restore_command
|
||||||
|
+ ('--batch',)
|
||||||
+ (
|
+ (
|
||||||
tuple(data_source['restore_options'].split(' '))
|
tuple(data_source['restore_options'].split(' '))
|
||||||
if 'restore_options' in data_source
|
if 'restore_options' in data_source
|
||||||
|
|
|
@ -142,6 +142,27 @@ def test_database_names_to_dump_runs_mariadb_with_list_options():
|
||||||
assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
|
assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
|
||||||
|
|
||||||
|
|
||||||
|
def test_database_names_to_dump_runs_non_default_mariadb_with_list_options():
|
||||||
|
database = {
|
||||||
|
'name': 'all',
|
||||||
|
'list_options': '--defaults-extra-file=mariadb.cnf',
|
||||||
|
'mariadb_command': 'custom_mariadb',
|
||||||
|
}
|
||||||
|
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
||||||
|
extra_environment=None,
|
||||||
|
full_command=(
|
||||||
|
'custom_mariadb', # Custom MariaDB command
|
||||||
|
'--defaults-extra-file=mariadb.cnf',
|
||||||
|
'--skip-column-names',
|
||||||
|
'--batch',
|
||||||
|
'--execute',
|
||||||
|
'show schemas',
|
||||||
|
),
|
||||||
|
).and_return(('foo\nbar')).once()
|
||||||
|
|
||||||
|
assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
|
||||||
|
|
||||||
|
|
||||||
def test_execute_dump_command_runs_mariadb_dump():
|
def test_execute_dump_command_runs_mariadb_dump():
|
||||||
process = flexmock()
|
process = flexmock()
|
||||||
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
|
||||||
|
@ -315,6 +336,44 @@ def test_execute_dump_command_runs_mariadb_dump_with_options():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execute_dump_command_runs_non_default_mariadb_dump_with_options():
|
||||||
|
process = flexmock()
|
||||||
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
|
||||||
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
||||||
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
|
(
|
||||||
|
'custom_mariadb_dump', # Custom MariaDB dump command
|
||||||
|
'--stuff=such',
|
||||||
|
'--add-drop-database',
|
||||||
|
'--databases',
|
||||||
|
'foo',
|
||||||
|
'--result-file',
|
||||||
|
'dump',
|
||||||
|
),
|
||||||
|
extra_environment=None,
|
||||||
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
|
assert (
|
||||||
|
module.execute_dump_command(
|
||||||
|
database={
|
||||||
|
'name': 'foo',
|
||||||
|
'mariadb_dump_command': 'custom_mariadb_dump',
|
||||||
|
'options': '--stuff=such',
|
||||||
|
}, # Custom MariaDB dump command specified
|
||||||
|
log_prefix='log',
|
||||||
|
dump_path=flexmock(),
|
||||||
|
database_names=('foo',),
|
||||||
|
extra_environment=None,
|
||||||
|
dry_run=False,
|
||||||
|
dry_run_label='',
|
||||||
|
)
|
||||||
|
== process
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump():
|
def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump():
|
||||||
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
|
||||||
flexmock(module.os.path).should_receive('exists').and_return(True)
|
flexmock(module.os.path).should_receive('exists').and_return(True)
|
||||||
|
@ -435,6 +494,36 @@ def test_restore_data_source_dump_runs_mariadb_with_options():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_data_source_dump_runs_non_default_mariadb_with_options():
|
||||||
|
hook_config = [
|
||||||
|
{'name': 'foo', 'restore_options': '--harder', 'mariadb_command': 'custom_mariadb'}
|
||||||
|
]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
('custom_mariadb', '--batch', '--harder'),
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment=None,
|
||||||
|
).once()
|
||||||
|
|
||||||
|
module.restore_data_source_dump(
|
||||||
|
hook_config,
|
||||||
|
{},
|
||||||
|
'test.yaml',
|
||||||
|
data_source=hook_config[0],
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port():
|
def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port():
|
||||||
hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
||||||
extract_process = flexmock(stdout=flexmock())
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
|
@ -142,6 +142,27 @@ def test_database_names_to_dump_runs_mysql_with_list_options():
|
||||||
assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
|
assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
|
||||||
|
|
||||||
|
|
||||||
|
def test_database_names_to_dump_runs_non_default_mysql_with_list_options():
|
||||||
|
database = {
|
||||||
|
'name': 'all',
|
||||||
|
'list_options': '--defaults-extra-file=my.cnf',
|
||||||
|
'mysql_command': 'custom_mysql',
|
||||||
|
}
|
||||||
|
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
||||||
|
extra_environment=None,
|
||||||
|
full_command=(
|
||||||
|
'custom_mysql', # Custom MySQL command
|
||||||
|
'--defaults-extra-file=my.cnf',
|
||||||
|
'--skip-column-names',
|
||||||
|
'--batch',
|
||||||
|
'--execute',
|
||||||
|
'show schemas',
|
||||||
|
),
|
||||||
|
).and_return(('foo\nbar')).once()
|
||||||
|
|
||||||
|
assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
|
||||||
|
|
||||||
|
|
||||||
def test_execute_dump_command_runs_mysqldump():
|
def test_execute_dump_command_runs_mysqldump():
|
||||||
process = flexmock()
|
process = flexmock()
|
||||||
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
|
||||||
|
@ -315,6 +336,42 @@ def test_execute_dump_command_runs_mysqldump_with_options():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execute_dump_command_runs_non_default_mysqldump():
|
||||||
|
process = flexmock()
|
||||||
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
|
||||||
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
||||||
|
flexmock(module.dump).should_receive('create_named_pipe_for_dump')
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
|
(
|
||||||
|
'custom_mysqldump', # Custom MySQL dump command
|
||||||
|
'--add-drop-database',
|
||||||
|
'--databases',
|
||||||
|
'foo',
|
||||||
|
'--result-file',
|
||||||
|
'dump',
|
||||||
|
),
|
||||||
|
extra_environment=None,
|
||||||
|
run_to_completion=False,
|
||||||
|
).and_return(process).once()
|
||||||
|
|
||||||
|
assert (
|
||||||
|
module.execute_dump_command(
|
||||||
|
database={
|
||||||
|
'name': 'foo',
|
||||||
|
'mysql_dump_command': 'custom_mysqldump',
|
||||||
|
}, # Custom MySQL dump command specified
|
||||||
|
log_prefix='log',
|
||||||
|
dump_path=flexmock(),
|
||||||
|
database_names=('foo',),
|
||||||
|
extra_environment=None,
|
||||||
|
dry_run=False,
|
||||||
|
dry_run_label='',
|
||||||
|
)
|
||||||
|
== process
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_execute_dump_command_with_duplicate_dump_skips_mysqldump():
|
def test_execute_dump_command_with_duplicate_dump_skips_mysqldump():
|
||||||
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
|
flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump')
|
||||||
flexmock(module.os.path).should_receive('exists').and_return(True)
|
flexmock(module.os.path).should_receive('exists').and_return(True)
|
||||||
|
@ -435,6 +492,34 @@ def test_restore_data_source_dump_runs_mysql_with_options():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_data_source_dump_runs_non_default_mysql_with_options():
|
||||||
|
hook_config = [{'name': 'foo', 'mysql_command': 'custom_mysql', 'restore_options': '--harder'}]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
('custom_mysql', '--batch', '--harder'),
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment=None,
|
||||||
|
).once()
|
||||||
|
|
||||||
|
module.restore_data_source_dump(
|
||||||
|
hook_config,
|
||||||
|
{},
|
||||||
|
'test.yaml',
|
||||||
|
data_source=hook_config[0],
|
||||||
|
dry_run=False,
|
||||||
|
extract_process=extract_process,
|
||||||
|
connection_params={
|
||||||
|
'hostname': None,
|
||||||
|
'port': None,
|
||||||
|
'username': None,
|
||||||
|
'password': None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_restore_data_source_dump_runs_mysql_with_hostname_and_port():
|
def test_restore_data_source_dump_runs_mysql_with_hostname_and_port():
|
||||||
hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
||||||
extract_process = flexmock(stdout=flexmock())
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
Loading…
Reference in a new issue