Add configuration options for database command customization (#630).
This commit is contained in:
parent
113c0e7616
commit
30cca62d09
9 changed files with 192 additions and 11 deletions
3
NEWS
3
NEWS
|
@ -5,6 +5,9 @@
|
||||||
* #602: Fix logs that interfere with JSON output by making warnings go to stderr instead of stdout.
|
* #602: Fix logs that interfere with JSON output by making warnings go to stderr instead of stdout.
|
||||||
* #622: Fix traceback when include merging configuration files on ARM64.
|
* #622: Fix traceback when include merging configuration files on ARM64.
|
||||||
* #629: Skip warning about excluded special files when no special files have been excluded.
|
* #629: Skip warning about excluded special files when no special files have been excluded.
|
||||||
|
* #630: Add configuration options for database command customization: "list_options",
|
||||||
|
"restore_options", and "analyze_options" for PostgreSQL, "restore_options" for MySQL, and
|
||||||
|
"restore_options" for MongoDB.
|
||||||
|
|
||||||
1.7.5
|
1.7.5
|
||||||
* #311: Override PostgreSQL dump/restore commands via configuration options.
|
* #311: Override PostgreSQL dump/restore commands via configuration options.
|
||||||
|
|
|
@ -806,6 +806,30 @@ properties:
|
||||||
any validation on them. See pg_dump
|
any validation on them. See pg_dump
|
||||||
documentation for details.
|
documentation for details.
|
||||||
example: --role=someone
|
example: --role=someone
|
||||||
|
list_options:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Additional psql options to pass directly to the
|
||||||
|
psql command that lists available databases,
|
||||||
|
without performing any validation on them. See
|
||||||
|
psql documentation for details.
|
||||||
|
example: --role=someone
|
||||||
|
restore_options:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Additional pg_restore/psql options to pass
|
||||||
|
directly to the restore command, without
|
||||||
|
performing any validation on them. See
|
||||||
|
pg_restore/psql documentation for details.
|
||||||
|
example: --role=someone
|
||||||
|
analyze_options:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Additional psql options to pass directly to the
|
||||||
|
analyze command run after a restore, without
|
||||||
|
performing any validation on them. See psql
|
||||||
|
documentation for details.
|
||||||
|
example: --role=someone
|
||||||
description: |
|
description: |
|
||||||
List of one or more PostgreSQL databases to dump before
|
List of one or more PostgreSQL databases to dump before
|
||||||
creating a backup, run once per configuration file. The
|
creating a backup, run once per configuration file. The
|
||||||
|
@ -868,14 +892,6 @@ properties:
|
||||||
file of that format, allowing more convenient
|
file of that format, allowing more convenient
|
||||||
restores of individual databases.
|
restores of individual databases.
|
||||||
example: directory
|
example: directory
|
||||||
list_options:
|
|
||||||
type: string
|
|
||||||
description: |
|
|
||||||
Additional mysql options to pass directly to
|
|
||||||
the mysql command that lists available
|
|
||||||
databases, without performing any validation on
|
|
||||||
them. See mysql documentation for details.
|
|
||||||
example: --defaults-extra-file=my.cnf
|
|
||||||
options:
|
options:
|
||||||
type: string
|
type: string
|
||||||
description: |
|
description: |
|
||||||
|
@ -884,6 +900,22 @@ properties:
|
||||||
validation on them. See mysqldump documentation
|
validation on them. See mysqldump documentation
|
||||||
for details.
|
for details.
|
||||||
example: --skip-comments
|
example: --skip-comments
|
||||||
|
list_options:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Additional mysql options to pass directly to
|
||||||
|
the mysql command that lists available
|
||||||
|
databases, without performing any validation on
|
||||||
|
them. See mysql documentation for details.
|
||||||
|
example: --defaults-extra-file=my.cnf
|
||||||
|
restore_options:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Additional mysql options to pass directly to
|
||||||
|
the mysql command that restores database dumps,
|
||||||
|
without performing any validation on them. See
|
||||||
|
mysql documentation for details.
|
||||||
|
example: --defaults-extra-file=my.cnf
|
||||||
description: |
|
description: |
|
||||||
List of one or more MySQL/MariaDB databases to dump before
|
List of one or more MySQL/MariaDB databases to dump before
|
||||||
creating a backup, run once per configuration file. The
|
creating a backup, run once per configuration file. The
|
||||||
|
@ -956,7 +988,15 @@ properties:
|
||||||
directly to the dump command, without performing
|
directly to the dump command, without performing
|
||||||
any validation on them. See mongodump
|
any validation on them. See mongodump
|
||||||
documentation for details.
|
documentation for details.
|
||||||
example: --role=someone
|
example: --dumpDbUsersAndRoles
|
||||||
|
restore_options:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Additional mongorestore options to pass
|
||||||
|
directly to the dump command, without performing
|
||||||
|
any validation on them. See mongorestore
|
||||||
|
documentation for details.
|
||||||
|
example: --restoreDbUsersAndRoles
|
||||||
description: |
|
description: |
|
||||||
List of one or more MongoDB databases to dump before
|
List of one or more MongoDB databases to dump before
|
||||||
creating a backup, run once per configuration file. The
|
creating a backup, run once per configuration file. The
|
||||||
|
|
|
@ -160,4 +160,6 @@ def build_restore_command(extract_process, database, dump_filename):
|
||||||
command.extend(('--password', database['password']))
|
command.extend(('--password', database['password']))
|
||||||
if 'authentication_database' in database:
|
if 'authentication_database' in database:
|
||||||
command.extend(('--authenticationDatabase', database['authentication_database']))
|
command.extend(('--authenticationDatabase', database['authentication_database']))
|
||||||
|
if 'restore_options' in database:
|
||||||
|
command.extend(database['restore_options'].split(' '))
|
||||||
return command
|
return command
|
||||||
|
|
|
@ -197,6 +197,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
database = database_config[0]
|
database = database_config[0]
|
||||||
restore_command = (
|
restore_command = (
|
||||||
('mysql', '--batch')
|
('mysql', '--batch')
|
||||||
|
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
|
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
|
||||||
|
|
|
@ -62,7 +62,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
+ (('--username', database['username']) if 'username' in database else ())
|
+ (('--username', database['username']) if 'username' in database else ())
|
||||||
+ (tuple(database['options'].split(' ')) if 'options' in database else ())
|
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
|
||||||
)
|
)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
'{}: Querying for "all" PostgreSQL databases to dump{}'.format(log_prefix, dry_run_label)
|
'{}: Querying for "all" PostgreSQL databases to dump{}'.format(log_prefix, dry_run_label)
|
||||||
|
@ -204,6 +204,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
+ (('--username', database['username']) if 'username' in database else ())
|
+ (('--username', database['username']) if 'username' in database else ())
|
||||||
+ (('--dbname', database['name']) if not all_databases else ())
|
+ (('--dbname', database['name']) if not all_databases else ())
|
||||||
|
+ (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
|
||||||
+ ('--command', 'ANALYZE')
|
+ ('--command', 'ANALYZE')
|
||||||
)
|
)
|
||||||
pg_restore_command = database.get('pg_restore_command') or 'pg_restore'
|
pg_restore_command = database.get('pg_restore_command') or 'pg_restore'
|
||||||
|
@ -217,6 +218,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
|
||||||
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
+ (('--host', database['hostname']) if 'hostname' in database else ())
|
||||||
+ (('--port', str(database['port'])) if 'port' in database else ())
|
+ (('--port', str(database['port'])) if 'port' in database else ())
|
||||||
+ (('--username', database['username']) if 'username' in database else ())
|
+ (('--username', database['username']) if 'username' in database else ())
|
||||||
|
+ (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
|
||||||
+ (() if extract_process else (dump_filename,))
|
+ (() if extract_process else (dump_filename,))
|
||||||
)
|
)
|
||||||
extra_environment = make_extra_environment(database)
|
extra_environment = make_extra_environment(database)
|
||||||
|
|
|
@ -76,6 +76,11 @@ hooks:
|
||||||
options: "--ssl"
|
options: "--ssl"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
See your [borgmatic configuration
|
||||||
|
file](https://torsion.org/borgmatic/docs/reference/configuration/) for
|
||||||
|
additional customization of the options passed to database commands (when
|
||||||
|
listing databases, restoring databases, etc.).
|
||||||
|
|
||||||
|
|
||||||
### All databases
|
### All databases
|
||||||
|
|
||||||
|
|
|
@ -256,6 +256,24 @@ def test_restore_database_dump_runs_mongorestore_with_username_and_password():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_runs_mongorestore_with_options():
|
||||||
|
database_config = [{'name': 'foo', 'restore_options': '--harder',}]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('make_dump_path')
|
||||||
|
flexmock(module.dump).should_receive('make_database_dump_filename')
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
['mongorestore', '--archive', '--drop', '--db', 'foo', '--harder',],
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
).once()
|
||||||
|
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_restore_database_dump_runs_psql_for_all_database_dump():
|
def test_restore_database_dump_runs_psql_for_all_database_dump():
|
||||||
database_config = [{'name': 'all'}]
|
database_config = [{'name': 'all'}]
|
||||||
extract_process = flexmock(stdout=flexmock())
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
|
@ -336,6 +336,23 @@ def test_restore_database_dump_errors_on_multiple_database_config():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_runs_mysql_with_options():
|
||||||
|
database_config = [{'name': 'foo', 'restore_options': '--harder'}]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
('mysql', '--batch', '--harder'),
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment=None,
|
||||||
|
).once()
|
||||||
|
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_restore_database_dump_runs_mysql_with_hostname_and_port():
|
def test_restore_database_dump_runs_mysql_with_hostname_and_port():
|
||||||
database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
|
||||||
extract_process = flexmock(stdout=flexmock())
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
|
@ -36,6 +36,55 @@ def test_database_names_to_dump_with_all_and_format_lists_databases():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port():
|
||||||
|
database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234}
|
||||||
|
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
||||||
|
(
|
||||||
|
'psql',
|
||||||
|
'--list',
|
||||||
|
'--no-password',
|
||||||
|
'--csv',
|
||||||
|
'--tuples-only',
|
||||||
|
'--host',
|
||||||
|
'localhost',
|
||||||
|
'--port',
|
||||||
|
'1234',
|
||||||
|
),
|
||||||
|
extra_environment=object,
|
||||||
|
).and_return('foo,test,\nbar,test,"stuff and such"')
|
||||||
|
|
||||||
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
|
||||||
|
'foo',
|
||||||
|
'bar',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_database_names_to_dump_with_all_and_format_lists_databases_with_username():
|
||||||
|
database = {'name': 'all', 'format': 'custom', 'username': 'postgres'}
|
||||||
|
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
||||||
|
('psql', '--list', '--no-password', '--csv', '--tuples-only', '--username', 'postgres'),
|
||||||
|
extra_environment=object,
|
||||||
|
).and_return('foo,test,\nbar,test,"stuff and such"')
|
||||||
|
|
||||||
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
|
||||||
|
'foo',
|
||||||
|
'bar',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_database_names_to_dump_with_all_and_format_lists_databases_with_options():
|
||||||
|
database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'}
|
||||||
|
flexmock(module).should_receive('execute_command_and_capture_output').with_args(
|
||||||
|
('psql', '--list', '--no-password', '--csv', '--tuples-only', '--harder'),
|
||||||
|
extra_environment=object,
|
||||||
|
).and_return('foo,test,\nbar,test,"stuff and such"')
|
||||||
|
|
||||||
|
assert module.database_names_to_dump(database, flexmock(), flexmock(), flexmock()) == (
|
||||||
|
'foo',
|
||||||
|
'bar',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_database_names_to_dump_with_all_and_format_excludes_particular_databases():
|
def test_database_names_to_dump_with_all_and_format_excludes_particular_databases():
|
||||||
database = {'name': 'all', 'format': 'custom'}
|
database = {'name': 'all', 'format': 'custom'}
|
||||||
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
|
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
|
||||||
|
@ -90,7 +139,7 @@ def test_dump_databases_raises_when_no_database_names_to_dump():
|
||||||
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
|
||||||
|
|
||||||
|
|
||||||
def test_dump_databases_with_dupliate_dump_skips_pg_dump():
|
def test_dump_databases_with_duplicate_dump_skips_pg_dump():
|
||||||
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
databases = [{'name': 'foo'}, {'name': 'bar'}]
|
||||||
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
||||||
flexmock(module).should_receive('make_dump_path').and_return('')
|
flexmock(module).should_receive('make_dump_path').and_return('')
|
||||||
|
@ -480,6 +529,50 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_restore_database_dump_runs_pg_restore_with_options():
|
||||||
|
database_config = [
|
||||||
|
{'name': 'foo', 'restore_options': '--harder', 'analyze_options': '--smarter'}
|
||||||
|
]
|
||||||
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
|
||||||
|
flexmock(module).should_receive('make_dump_path')
|
||||||
|
flexmock(module.dump).should_receive('make_database_dump_filename')
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
(
|
||||||
|
'pg_restore',
|
||||||
|
'--no-password',
|
||||||
|
'--if-exists',
|
||||||
|
'--exit-on-error',
|
||||||
|
'--clean',
|
||||||
|
'--dbname',
|
||||||
|
'foo',
|
||||||
|
'--harder',
|
||||||
|
),
|
||||||
|
processes=[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
extra_environment={'PGSSLMODE': 'disable'},
|
||||||
|
).once()
|
||||||
|
flexmock(module).should_receive('execute_command').with_args(
|
||||||
|
(
|
||||||
|
'psql',
|
||||||
|
'--no-password',
|
||||||
|
'--quiet',
|
||||||
|
'--dbname',
|
||||||
|
'foo',
|
||||||
|
'--smarter',
|
||||||
|
'--command',
|
||||||
|
'ANALYZE',
|
||||||
|
),
|
||||||
|
extra_environment={'PGSSLMODE': 'disable'},
|
||||||
|
).once()
|
||||||
|
|
||||||
|
module.restore_database_dump(
|
||||||
|
database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_restore_database_dump_runs_psql_for_all_database_dump():
|
def test_restore_database_dump_runs_psql_for_all_database_dump():
|
||||||
database_config = [{'name': 'all'}]
|
database_config = [{'name': 'all'}]
|
||||||
extract_process = flexmock(stdout=flexmock())
|
extract_process = flexmock(stdout=flexmock())
|
||||||
|
|
Loading…
Reference in a new issue