Add configuration options for database command customization (#630).
This commit is contained in:
parent
113c0e7616
commit
30cca62d09
9 changed files with 192 additions and 11 deletions
3
NEWS
3
NEWS
|
@ -5,6 +5,9 @@
|
|||
* #602: Fix logs that interfere with JSON output by making warnings go to stderr instead of stdout.
|
||||
* #622: Fix traceback when include merging configuration files on ARM64.
|
||||
* #629: Skip warning about excluded special files when no special files have been excluded.
|
||||
* #630: Add configuration options for database command customization: "list_options",
|
||||
"restore_options", and "analyze_options" for PostgreSQL, "restore_options" for MySQL, and
|
||||
"restore_options" for MongoDB.
|
||||
|
||||
1.7.5
|
||||
* #311: Override PostgreSQL dump/restore commands via configuration options.
|
||||
|
|
|
@ -806,6 +806,30 @@ properties:
|
|||
any validation on them. See pg_dump
|
||||
documentation for details.
|
||||
example: --role=someone
|
||||
list_options:
|
||||
type: string
|
||||
description: |
|
||||
Additional psql options to pass directly to the
|
||||
psql command that lists available databases,
|
||||
without performing any validation on them. See
|
||||
psql documentation for details.
|
||||
example: --role=someone
|
||||
restore_options:
|
||||
type: string
|
||||
description: |
|
||||
Additional pg_restore/psql options to pass
|
||||
directly to the restore command, without
|
||||
performing any validation on them. See
|
||||
pg_restore/psql documentation for details.
|
||||
example: --role=someone
|
||||
analyze_options:
|
||||
type: string
|
||||
description: |
|
||||
Additional psql options to pass directly to the
|
||||
analyze command run after a restore, without
|
||||
performing any validation on them. See psql
|
||||
documentation for details.
|
||||
example: --role=someone
|
||||
description: |
|
||||
List of one or more PostgreSQL databases to dump before
|
||||
creating a backup, run once per configuration file. The
|
||||
|
@ -868,14 +892,6 @@ properties:
|
|||
file of that format, allowing more convenient
|
||||
restores of individual databases.
|
||||
example: directory
|
||||
list_options:
|
||||
type: string
|
||||
description: |
|
||||
Additional mysql options to pass directly to
|
||||
the mysql command that lists available
|
||||
databases, without performing any validation on
|
||||
them. See mysql documentation for details.
|
||||
example: --defaults-extra-file=my.cnf
|
||||
options:
|
||||
type: string
|
||||
description: |
|
||||
|
@ -884,6 +900,22 @@ properties:
|
|||
validation on them. See mysqldump documentation
|
||||
for details.
|
||||
example: --skip-comments
|
||||
list_options:
|
||||
type: string
|
||||
description: |
|
||||
Additional mysql options to pass directly to
|
||||
the mysql command that lists available
|
||||
databases, without performing any validation on
|
||||
them. See mysql documentation for details.
|
||||
example: --defaults-extra-file=my.cnf
|
||||
restore_options:
|
||||
type: string
|
||||
description: |
|
||||
Additional mysql options to pass directly to
|
||||
the mysql command that restores database dumps,
|
||||
without performing any validation on them. See
|
||||
mysql documentation for details.
|
||||
example: --defaults-extra-file=my.cnf
|
||||
description: |
|
||||
List of one or more MySQL/MariaDB databases to dump before
|
||||
creating a backup, run once per configuration file. The
|
||||
|
@ -956,7 +988,15 @@ properties:
|
|||
directly to the dump command, without performing
|
||||
any validation on them. See mongodump
|
||||
documentation for details.
|
||||
example: --role=someone
|
||||
example: --dumpDbUsersAndRoles
|
||||
restore_options:
|
||||
type: string
|
||||
description: |
|
||||
Additional mongorestore options to pass
|
||||
directly to the dump command, without performing
|
||||
any validation on them. See mongorestore
|
||||
documentation for details.
|
||||
example: --restoreDbUsersAndRoles
|
||||
description: |
|
||||
List of one or more MongoDB databases to dump before
|
||||
creating a backup, run once per configuration file. The
|
||||
|
|
|
@ -160,4 +160,6 @@ def build_restore_command(extract_process, database, dump_filename):
|
|||
command.extend(('--password', database['password']))
|
||||
if 'authentication_database' in database:
|
||||
command.extend(('--authenticationDatabase', database['authentication_database']))
|
||||
if 'restore_options' in database:
|
||||
command.extend(database['restore_options'].split(' '))
|
||||
return command
|
||||
|
|
|
@ -197,6 +197,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
|||
database = database_config[0]
|
||||
restore_command = (
|
||||
('mysql', '--batch')
|
||||
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
|
||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
|
||||
|
|
|
@ -62,7 +62,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe
|
|||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||
+ (('--username', database['username']) if 'username' in database else ())
|
||||
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
||||
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
|
||||
)
|
||||
logger.debug(
|
||||
'{}: Querying for "all" PostgreSQL databases to dump{}'.format(log_prefix, dry_run_label)
|
||||
|
@ -204,6 +204,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
|||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||
+ (('--username', database['username']) if 'username' in database else ())
|
||||
+ (('--dbname', database['name']) if not all_databases else ())
|
||||
+ (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
|
||||
+ ('--command', 'ANALYZE')
|
||||
)
|
||||
pg_restore_command = database.get('pg_restore_command') or 'pg_restore'
|
||||
|
@ -217,6 +218,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
|||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||
+ (('--username', database['username']) if 'username' in database else ())
|
||||
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
|
||||
+ (() if extract_process else (dump_filename,))
|
||||
)
|
||||
extra_environment = make_extra_environment(database)
|
||||
|
|
|
@ -76,6 +76,11 @@ hooks:
|
|||
options: "--ssl"
|
||||
```
|
||||
|
||||
See your [borgmatic configuration
|
||||
file](https://torsion.org/borgmatic/docs/reference/configuration/) for
|
||||
additional customization of the options passed to database commands (when
|
||||
listing databases, restoring databases, etc.).
|
||||
|
||||
|
||||
### All databases
|
||||
|
||||
|
|
|
@ -256,6 +256,24 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
|
|||
)
|
||||
|
||||
|
||||
def test_restore_database_dump_runs_mongorestore_with_options():
|
||||
database_config = [{'name': 'foo', 'restore_options': '--harder',}]
|
||||
extract_process = flexmock(stdout=flexmock())
|
||||
|
||||
flexmock(module).should_receive('make_dump_path')
|
||||
flexmock(module.dump).should_receive('make_database_dump_filename')
|
||||
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||
['mongorestore', '--archive', '--drop', '--db', 'foo', '--harder',],
|
||||
processes=[extract_process],
|
||||
output_log_level=logging.DEBUG,
|
||||
input_file=extract_process.stdout,
|
||||
).once()
|
||||
|
||||
module.restore_database_dump(
|
||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||
)
|
||||
|
||||
|
||||
def test_restore_database_dump_runs_psql_for_all_database_dump():
|
||||
database_config = [{'name': 'all'}]
|
||||
extract_process = flexmock(stdout=flexmock())
|
||||
|
|
|
@ -336,6 +336,23 @@ def test_restore_database_dump_errors_on_multiple_database_config():
|
|||
)
|
||||
|
||||
|
||||
def test_restore_database_dump_runs_mysql_with_options():
|
||||
database_config = [{'name': 'foo', 'restore_options': '--harder'}]
|
||||
extract_process = flexmock(stdout=flexmock())
|
||||
|
||||
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||
('mysql', '--batch', '--harder'),
|
||||
processes=[extract_process],
|
||||
output_log_level=logging.DEBUG,
|
||||
input_file=extract_process.stdout,
|
||||
extra_environment=None,
|
||||
).once()
|
||||
|
||||
module.restore_database_dump(
|
||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||
)
|
||||
|
||||
|
||||
def test_restore_database_dump_runs_mysql_with_hostname_and_port():
|
||||
database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
||||
extract_process = flexmock(stdout=flexmock())
|
||||
|
|
|
@ -36,6 +36,55 @@ def test_database_names_to_dump_with_all_and_format_lists_databases():
|
|||
)
|
||||
|
||||
|
||||
def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port():
|
||||
database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234}
|
||||
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
||||
(
|
||||
'psql',
|
||||
'--list',
|
||||
'--no-password',
|
||||
'--csv',
|
||||
'--tuples-only',
|
||||
'--host',
|
||||
'localhost',
|
||||
'--port',
|
||||
'1234',
|
||||
),
|
||||
extra_environment=object,
|
||||
).and_return('foo,test,\nbar,test,"stuff and such"')
|
||||
|
||||
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
|
||||
'foo',
|
||||
'bar',
|
||||
)
|
||||
|
||||
|
||||
def test_database_names_to_dump_with_all_and_format_lists_databases_with_username():
|
||||
database = {'name': 'all', 'format': 'custom', 'username': 'postgres'}
|
||||
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
||||
('psql', '--list', '--no-password', '--csv', '--tuples-only', '--username', 'postgres'),
|
||||
extra_environment=object,
|
||||
).and_return('foo,test,\nbar,test,"stuff and such"')
|
||||
|
||||
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
|
||||
'foo',
|
||||
'bar',
|
||||
)
|
||||
|
||||
|
||||
def test_database_names_to_dump_with_all_and_format_lists_databases_with_options():
|
||||
database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'}
|
||||
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
||||
('psql', '--list', '--no-password', '--csv', '--tuples-only', '--harder'),
|
||||
extra_environment=object,
|
||||
).and_return('foo,test,\nbar,test,"stuff and such"')
|
||||
|
||||
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
|
||||
'foo',
|
||||
'bar',
|
||||
)
|
||||
|
||||
|
||||
def test_database_names_to_dump_with_all_and_format_excludes_particular_databases():
|
||||
database = {'name': 'all', 'format': 'custom'}
|
||||
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
|
||||
|
@ -90,7 +139,7 @@ def test_dump_databases_raises_when_no_database_names_to_dump():
|
|||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
||||
|
||||
|
||||
def test_dump_databases_with_dupliate_dump_skips_pg_dump():
|
||||
def test_dump_databases_with_duplicate_dump_skips_pg_dump():
|
||||
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
||||
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||
|
@ -480,6 +529,50 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
|
|||
)
|
||||
|
||||
|
||||
def test_restore_database_dump_runs_pg_restore_with_options():
|
||||
database_config = [
|
||||
{'name': 'foo', 'restore_options': '--harder', 'analyze_options': '--smarter'}
|
||||
]
|
||||
extract_process = flexmock(stdout=flexmock())
|
||||
|
||||
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
||||
flexmock(module).should_receive('make_dump_path')
|
||||
flexmock(module.dump).should_receive('make_database_dump_filename')
|
||||
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||
(
|
||||
'pg_restore',
|
||||
'--no-password',
|
||||
'--if-exists',
|
||||
'--exit-on-error',
|
||||
'--clean',
|
||||
'--dbname',
|
||||
'foo',
|
||||
'--harder',
|
||||
),
|
||||
processes=[extract_process],
|
||||
output_log_level=logging.DEBUG,
|
||||
input_file=extract_process.stdout,
|
||||
extra_environment={'PGSSLMODE': 'disable'},
|
||||
).once()
|
||||
flexmock(module).should_receive('execute_command').with_args(
|
||||
(
|
||||
'psql',
|
||||
'--no-password',
|
||||
'--quiet',
|
||||
'--dbname',
|
||||
'foo',
|
||||
'--smarter',
|
||||
'--command',
|
||||
'ANALYZE',
|
||||
),
|
||||
extra_environment={'PGSSLMODE': 'disable'},
|
||||
).once()
|
||||
|
||||
module.restore_database_dump(
|
||||
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||
)
|
||||
|
||||
|
||||
def test_restore_database_dump_runs_psql_for_all_database_dump():
|
||||
database_config = [{'name': 'all'}]
|
||||
extract_process = flexmock(stdout=flexmock())
|
||||
|
|
Loading…
Reference in a new issue