Add configuration options for database command customization (#630).

This commit is contained in:
Dan Helfman 2023-01-26 14:59:17 -08:00
parent 113c0e7616
commit 30cca62d09
9 changed files with 192 additions and 11 deletions

3
NEWS
View file

@ -5,6 +5,9 @@
* #602: Fix logs that interfere with JSON output by making warnings go to stderr instead of stdout. * #602: Fix logs that interfere with JSON output by making warnings go to stderr instead of stdout.
* #622: Fix traceback when include merging configuration files on ARM64. * #622: Fix traceback when include merging configuration files on ARM64.
* #629: Skip warning about excluded special files when no special files have been excluded. * #629: Skip warning about excluded special files when no special files have been excluded.
* #630: Add configuration options for database command customization: "list_options",
"restore_options", and "analyze_options" for PostgreSQL, "restore_options" for MySQL, and
"restore_options" for MongoDB.
1.7.5 1.7.5
* #311: Override PostgreSQL dump/restore commands via configuration options. * #311: Override PostgreSQL dump/restore commands via configuration options.

View file

@ -806,6 +806,30 @@ properties:
any validation on them. See pg_dump any validation on them. See pg_dump
documentation for details. documentation for details.
example: --role=someone example: --role=someone
list_options:
type: string
description: |
Additional psql options to pass directly to the
psql command that lists available databases,
without performing any validation on them. See
psql documentation for details.
example: --role=someone
restore_options:
type: string
description: |
Additional pg_restore/psql options to pass
directly to the restore command, without
performing any validation on them. See
pg_restore/psql documentation for details.
example: --role=someone
analyze_options:
type: string
description: |
Additional psql options to pass directly to the
analyze command run after a restore, without
performing any validation on them. See psql
documentation for details.
example: --role=someone
description: | description: |
List of one or more PostgreSQL databases to dump before List of one or more PostgreSQL databases to dump before
creating a backup, run once per configuration file. The creating a backup, run once per configuration file. The
@ -868,14 +892,6 @@ properties:
file of that format, allowing more convenient file of that format, allowing more convenient
restores of individual databases. restores of individual databases.
example: directory example: directory
list_options:
type: string
description: |
Additional mysql options to pass directly to
the mysql command that lists available
databases, without performing any validation on
them. See mysql documentation for details.
example: --defaults-extra-file=my.cnf
options: options:
type: string type: string
description: | description: |
@ -884,6 +900,22 @@ properties:
validation on them. See mysqldump documentation validation on them. See mysqldump documentation
for details. for details.
example: --skip-comments example: --skip-comments
list_options:
type: string
description: |
Additional mysql options to pass directly to
the mysql command that lists available
databases, without performing any validation on
them. See mysql documentation for details.
example: --defaults-extra-file=my.cnf
restore_options:
type: string
description: |
Additional mysql options to pass directly to
the mysql command that restores database dumps,
without performing any validation on them. See
mysql documentation for details.
example: --defaults-extra-file=my.cnf
description: | description: |
List of one or more MySQL/MariaDB databases to dump before List of one or more MySQL/MariaDB databases to dump before
creating a backup, run once per configuration file. The creating a backup, run once per configuration file. The
@ -956,7 +988,15 @@ properties:
directly to the dump command, without performing directly to the dump command, without performing
any validation on them. See mongodump any validation on them. See mongodump
documentation for details. documentation for details.
example: --role=someone example: --dumpDbUsersAndRoles
restore_options:
type: string
description: |
Additional mongorestore options to pass
directly to the dump command, without performing
any validation on them. See mongorestore
documentation for details.
example: --restoreDbUsersAndRoles
description: | description: |
List of one or more MongoDB databases to dump before List of one or more MongoDB databases to dump before
creating a backup, run once per configuration file. The creating a backup, run once per configuration file. The

View file

@ -160,4 +160,6 @@ def build_restore_command(extract_process, database, dump_filename):
command.extend(('--password', database['password'])) command.extend(('--password', database['password']))
if 'authentication_database' in database: if 'authentication_database' in database:
command.extend(('--authenticationDatabase', database['authentication_database'])) command.extend(('--authenticationDatabase', database['authentication_database']))
if 'restore_options' in database:
command.extend(database['restore_options'].split(' '))
return command return command

View file

@ -197,6 +197,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
database = database_config[0] database = database_config[0]
restore_command = ( restore_command = (
('mysql', '--batch') ('mysql', '--batch')
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
+ (('--host', database['hostname']) if 'hostname' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())

View file

@ -62,7 +62,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe
+ (('--host', database['hostname']) if 'hostname' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ()) + (('--username', database['username']) if 'username' in database else ())
+ (tuple(database['options'].split(' ')) if 'options' in database else ()) + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
) )
logger.debug( logger.debug(
'{}: Querying for "all" PostgreSQL databases to dump{}'.format(log_prefix, dry_run_label) '{}: Querying for "all" PostgreSQL databases to dump{}'.format(log_prefix, dry_run_label)
@ -204,6 +204,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
+ (('--port', str(database['port'])) if 'port' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ()) + (('--username', database['username']) if 'username' in database else ())
+ (('--dbname', database['name']) if not all_databases else ()) + (('--dbname', database['name']) if not all_databases else ())
+ (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
+ ('--command', 'ANALYZE') + ('--command', 'ANALYZE')
) )
pg_restore_command = database.get('pg_restore_command') or 'pg_restore' pg_restore_command = database.get('pg_restore_command') or 'pg_restore'
@ -217,6 +218,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
+ (('--host', database['hostname']) if 'hostname' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--port', str(database['port'])) if 'port' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ())
+ (('--username', database['username']) if 'username' in database else ()) + (('--username', database['username']) if 'username' in database else ())
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
+ (() if extract_process else (dump_filename,)) + (() if extract_process else (dump_filename,))
) )
extra_environment = make_extra_environment(database) extra_environment = make_extra_environment(database)

View file

@ -76,6 +76,11 @@ hooks:
options: "--ssl" options: "--ssl"
``` ```
See your [borgmatic configuration
file](https://torsion.org/borgmatic/docs/reference/configuration/) for
additional customization of the options passed to database commands (when
listing databases, restoring databases, etc.).
### All databases ### All databases

View file

@ -256,6 +256,24 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
) )
def test_restore_database_dump_runs_mongorestore_with_options():
database_config = [{'name': 'foo', 'restore_options': '--harder',}]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
['mongorestore', '--archive', '--drop', '--db', 'foo', '--harder',],
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
)
def test_restore_database_dump_runs_psql_for_all_database_dump(): def test_restore_database_dump_runs_psql_for_all_database_dump():
database_config = [{'name': 'all'}] database_config = [{'name': 'all'}]
extract_process = flexmock(stdout=flexmock()) extract_process = flexmock(stdout=flexmock())

View file

@ -336,6 +336,23 @@ def test_restore_database_dump_errors_on_multiple_database_config():
) )
def test_restore_database_dump_runs_mysql_with_options():
database_config = [{'name': 'foo', 'restore_options': '--harder'}]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('execute_command_with_processes').with_args(
('mysql', '--batch', '--harder'),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment=None,
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
)
def test_restore_database_dump_runs_mysql_with_hostname_and_port(): def test_restore_database_dump_runs_mysql_with_hostname_and_port():
database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
extract_process = flexmock(stdout=flexmock()) extract_process = flexmock(stdout=flexmock())

View file

@ -36,6 +36,55 @@ def test_database_names_to_dump_with_all_and_format_lists_databases():
) )
def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port():
database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234}
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
(
'psql',
'--list',
'--no-password',
'--csv',
'--tuples-only',
'--host',
'localhost',
'--port',
'1234',
),
extra_environment=object,
).and_return('foo,test,\nbar,test,"stuff and such"')
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
'foo',
'bar',
)
def test_database_names_to_dump_with_all_and_format_lists_databases_with_username():
database = {'name': 'all', 'format': 'custom', 'username': 'postgres'}
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
('psql', '--list', '--no-password', '--csv', '--tuples-only', '--username', 'postgres'),
extra_environment=object,
).and_return('foo,test,\nbar,test,"stuff and such"')
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
'foo',
'bar',
)
def test_database_names_to_dump_with_all_and_format_lists_databases_with_options():
database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'}
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
('psql', '--list', '--no-password', '--csv', '--tuples-only', '--harder'),
extra_environment=object,
).and_return('foo,test,\nbar,test,"stuff and such"')
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
'foo',
'bar',
)
def test_database_names_to_dump_with_all_and_format_excludes_particular_databases(): def test_database_names_to_dump_with_all_and_format_excludes_particular_databases():
database = {'name': 'all', 'format': 'custom'} database = {'name': 'all', 'format': 'custom'}
flexmock(module).should_receive('execute_command_and_capture_output').and_return( flexmock(module).should_receive('execute_command_and_capture_output').and_return(
@ -90,7 +139,7 @@ def test_dump_databases_raises_when_no_database_names_to_dump():
module.dump_databases(databases, 'test.yaml', {}, dry_run=False) module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
def test_dump_databases_with_dupliate_dump_skips_pg_dump(): def test_dump_databases_with_duplicate_dump_skips_pg_dump():
databases = [{'name': 'foo'}, {'name': 'bar'}] databases = [{'name': 'foo'}, {'name': 'bar'}]
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('make_dump_path').and_return('')
@ -480,6 +529,50 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
) )
def test_restore_database_dump_runs_pg_restore_with_options():
database_config = [
{'name': 'foo', 'restore_options': '--harder', 'analyze_options': '--smarter'}
]
extract_process = flexmock(stdout=flexmock())
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
flexmock(module).should_receive('make_dump_path')
flexmock(module.dump).should_receive('make_database_dump_filename')
flexmock(module).should_receive('execute_command_with_processes').with_args(
(
'pg_restore',
'--no-password',
'--if-exists',
'--exit-on-error',
'--clean',
'--dbname',
'foo',
'--harder',
),
processes=[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
extra_environment={'PGSSLMODE': 'disable'},
).once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
'--no-password',
'--quiet',
'--dbname',
'foo',
'--smarter',
'--command',
'ANALYZE',
),
extra_environment={'PGSSLMODE': 'disable'},
).once()
module.restore_database_dump(
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
)
def test_restore_database_dump_runs_psql_for_all_database_dump(): def test_restore_database_dump_runs_psql_for_all_database_dump():
database_config = [{'name': 'all'}] database_config = [{'name': 'all'}]
extract_process = flexmock(stdout=flexmock()) extract_process = flexmock(stdout=flexmock())