feat: add dump-restore support for sqlite databases
This commit is contained in:
parent
2a306bef12
commit
af1cc27988
7 changed files with 244 additions and 5 deletions
|
@ -931,6 +931,31 @@ properties:
|
||||||
mysqldump/mysql commands (from either MySQL or MariaDB). See
|
mysqldump/mysql commands (from either MySQL or MariaDB). See
|
||||||
https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
|
https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
|
||||||
https://mariadb.com/kb/en/library/mysqldump/ for details.
|
https://mariadb.com/kb/en/library/mysqldump/ for details.
|
||||||
|
sqlite_databases:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: object
|
||||||
|
required: ['path','name']
|
||||||
|
additionalProperties: false
|
||||||
|
properties:
|
||||||
|
path:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Path to the SQLite database file to dump. If
|
||||||
|
relative, it is relative to the current working
|
||||||
|
directory. If absolute, it is relative to the
|
||||||
|
root of the filesystem. Note that using this
|
||||||
|
database hook implicitly enables both
|
||||||
|
read_special and one_file_system (see above) to
|
||||||
|
support dump and restore streaming.
|
||||||
|
example: /var/lib/sqlite/users.db
|
||||||
|
name:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
This is used to tag the database dump file with
|
||||||
|
a name. It is not used to identify the database
|
||||||
|
file itself.
|
||||||
|
example: users
|
||||||
mongodb_databases:
|
mongodb_databases:
|
||||||
type: array
|
type: array
|
||||||
items:
|
items:
|
||||||
|
|
|
@ -9,6 +9,7 @@ from borgmatic.hooks import (
|
||||||
ntfy,
|
ntfy,
|
||||||
pagerduty,
|
pagerduty,
|
||||||
postgresql,
|
postgresql,
|
||||||
|
sqlite,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -22,6 +23,7 @@ HOOK_NAME_TO_MODULE = {
|
||||||
'ntfy': ntfy,
|
'ntfy': ntfy,
|
||||||
'pagerduty': pagerduty,
|
'pagerduty': pagerduty,
|
||||||
'postgresql_databases': postgresql,
|
'postgresql_databases': postgresql,
|
||||||
|
'sqlite_databases': sqlite,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,12 @@ from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
DATABASE_HOOK_NAMES = ('postgresql_databases', 'mysql_databases', 'mongodb_databases')
|
DATABASE_HOOK_NAMES = (
|
||||||
|
'postgresql_databases',
|
||||||
|
'mysql_databases',
|
||||||
|
'mongodb_databases',
|
||||||
|
'sqlite_databases',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def make_database_dump_path(borgmatic_source_directory, database_hook_name):
|
def make_database_dump_path(borgmatic_source_directory, database_hook_name):
|
||||||
|
|
122
borgmatic/hooks/sqlite.py
Normal file
122
borgmatic/hooks/sqlite.py
Normal file
|
@ -0,0 +1,122 @@
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from subprocess import CalledProcessError
|
||||||
|
|
||||||
|
from borgmatic.execute import execute_command, execute_command_with_processes
|
||||||
|
from borgmatic.hooks import dump
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def make_dump_path(location_config): # pragma: no cover
|
||||||
|
'''
|
||||||
|
Make the dump path from the given location configuration and the name of this hook.
|
||||||
|
'''
|
||||||
|
return dump.make_database_dump_path(
|
||||||
|
location_config.get('borgmatic_source_directory'), 'sqlite_databases'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def dump_databases(databases, log_prefix, location_config, dry_run): # pragma: no cover
|
||||||
|
'''
|
||||||
|
Dump the given SQLite3 databases to a file. The databases are supplied as a sequence of
|
||||||
|
configuration dicts, as per the configuration schema. Use the given log prefix in any log
|
||||||
|
entries. Use the given location configuration dict to construct the destination path. If this
|
||||||
|
is a dry run, then don't actually dump anything.
|
||||||
|
'''
|
||||||
|
dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
|
||||||
|
processes = []
|
||||||
|
|
||||||
|
logger.info('{}: Dumping SQLite databases{}'.format(log_prefix, dry_run_label))
|
||||||
|
|
||||||
|
for database in databases:
|
||||||
|
database_path = database['path']
|
||||||
|
database_filename = database['name']
|
||||||
|
dump_path = make_dump_path(location_config)
|
||||||
|
dump_filename = dump.make_database_dump_filename(dump_path, database_filename)
|
||||||
|
if os.path.exists(dump_filename):
|
||||||
|
logger.warning(
|
||||||
|
f'{log_prefix}: Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}'
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
command = (
|
||||||
|
'sqlite3',
|
||||||
|
database_path,
|
||||||
|
'.dump',
|
||||||
|
'>',
|
||||||
|
dump_filename,
|
||||||
|
)
|
||||||
|
logger.debug(
|
||||||
|
f'{log_prefix}: Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}'
|
||||||
|
)
|
||||||
|
if dry_run:
|
||||||
|
continue
|
||||||
|
|
||||||
|
dump.create_parent_directory_for_dump(dump_filename)
|
||||||
|
processes.append(execute_command(command, shell=True, run_to_completion=False))
|
||||||
|
|
||||||
|
return processes
|
||||||
|
|
||||||
|
|
||||||
|
def remove_database_dumps(databases, log_prefix, location_config, dry_run): # pragma: no cover
|
||||||
|
'''
|
||||||
|
Remove the given SQLite3 database dumps from the filesystem. The databases are supplied as a
|
||||||
|
sequence of configuration dicts, as per the configuration schema. Use the given log prefix in
|
||||||
|
any log entries. Use the given location configuration dict to construct the destination path.
|
||||||
|
If this is a dry run, then don't actually remove anything.
|
||||||
|
'''
|
||||||
|
dump.remove_database_dumps(make_dump_path(location_config), 'SQLite', log_prefix, dry_run)
|
||||||
|
|
||||||
|
|
||||||
|
def make_database_dump_pattern(
|
||||||
|
databases, log_prefix, location_config, name=None
|
||||||
|
): # pragma: no cover
|
||||||
|
'''
|
||||||
|
Make a pattern that matches the given SQLite3 databases. The databases are supplied as a
|
||||||
|
sequence of configuration dicts, as per the configuration schema.
|
||||||
|
'''
|
||||||
|
return dump.make_database_dump_filename(make_dump_path(location_config), name)
|
||||||
|
|
||||||
|
|
||||||
|
def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
|
||||||
|
'''
|
||||||
|
Restore the given SQLite3 database from an extract stream. The database is supplied as a
|
||||||
|
one-element sequence containing a dict describing the database, as per the configuration schema.
|
||||||
|
Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
|
||||||
|
anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
|
||||||
|
output to consume.
|
||||||
|
'''
|
||||||
|
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
|
||||||
|
|
||||||
|
if len(database_config) != 1:
|
||||||
|
raise ValueError('The database configuration value is invalid')
|
||||||
|
|
||||||
|
database_path = database_config[0]['path']
|
||||||
|
|
||||||
|
logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
|
||||||
|
if dry_run:
|
||||||
|
return
|
||||||
|
|
||||||
|
remove_command = (
|
||||||
|
'rm',
|
||||||
|
database_path,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
execute_command(remove_command, shell=True)
|
||||||
|
except CalledProcessError:
|
||||||
|
logger.info(f'{log_prefix}: Database does not exist at {database_path}, skipping removal')
|
||||||
|
|
||||||
|
restore_command = (
|
||||||
|
'sqlite3',
|
||||||
|
database_path,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
|
||||||
|
# if the restore paths don't exist in the archive.
|
||||||
|
execute_command_with_processes(
|
||||||
|
restore_command,
|
||||||
|
[extract_process],
|
||||||
|
output_log_level=logging.DEBUG,
|
||||||
|
input_file=extract_process.stdout,
|
||||||
|
)
|
|
@ -15,8 +15,8 @@ consistent snapshot that is more suited for backups.
|
||||||
|
|
||||||
Fortunately, borgmatic includes built-in support for creating database dumps
|
Fortunately, borgmatic includes built-in support for creating database dumps
|
||||||
prior to running backups. For example, here is everything you need to dump and
|
prior to running backups. For example, here is everything you need to dump and
|
||||||
backup a couple of local PostgreSQL databases, a MySQL/MariaDB database, and a
|
backup a couple of local PostgreSQL databases, a MySQL/MariaDB database, a
|
||||||
MongoDB database:
|
MongoDB database and a SQLite database:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
hooks:
|
hooks:
|
||||||
|
@ -27,6 +27,9 @@ hooks:
|
||||||
- name: posts
|
- name: posts
|
||||||
mongodb_databases:
|
mongodb_databases:
|
||||||
- name: messages
|
- name: messages
|
||||||
|
sqlite_databases:
|
||||||
|
- path: /var/lib/sqlite3/mydb.sqlite
|
||||||
|
- name: mydb
|
||||||
```
|
```
|
||||||
|
|
||||||
As part of each backup, borgmatic streams a database dump for each configured
|
As part of each backup, borgmatic streams a database dump for each configured
|
||||||
|
@ -74,6 +77,9 @@ hooks:
|
||||||
password: trustsome1
|
password: trustsome1
|
||||||
authentication_database: mongousers
|
authentication_database: mongousers
|
||||||
options: "--ssl"
|
options: "--ssl"
|
||||||
|
sqlite_databases:
|
||||||
|
- path: /var/lib/sqlite3/mydb.sqlite
|
||||||
|
- name: mydb
|
||||||
```
|
```
|
||||||
|
|
||||||
See your [borgmatic configuration
|
See your [borgmatic configuration
|
||||||
|
@ -97,7 +103,8 @@ hooks:
|
||||||
```
|
```
|
||||||
|
|
||||||
Note that you may need to use a `username` of the `postgres` superuser for
|
Note that you may need to use a `username` of the `postgres` superuser for
|
||||||
this to work with PostgreSQL.
|
this to work with PostgreSQL. Also, the `all` database name is not supported
|
||||||
|
for SQLite databases.
|
||||||
|
|
||||||
<span class="minilink minilink-addedin">New in version 1.7.6</span> With
|
<span class="minilink minilink-addedin">New in version 1.7.6</span> With
|
||||||
PostgreSQL and MySQL, you can optionally dump "all" databases to separate
|
PostgreSQL and MySQL, you can optionally dump "all" databases to separate
|
||||||
|
@ -154,7 +161,7 @@ bring back any missing configuration files in order to restore a database.
|
||||||
|
|
||||||
## Supported databases
|
## Supported databases
|
||||||
|
|
||||||
As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, and MongoDB databases
|
As of now, borgmatic supports PostgreSQL, MySQL/MariaDB, MongoDB databases and SQLite databases
|
||||||
directly. But see below about general-purpose preparation and cleanup hooks as
|
directly. But see below about general-purpose preparation and cleanup hooks as
|
||||||
a work-around with other database systems. Also, please [file a
|
a work-around with other database systems. Also, please [file a
|
||||||
ticket](https://torsion.org/borgmatic/#issues) for additional database systems
|
ticket](https://torsion.org/borgmatic/#issues) for additional database systems
|
||||||
|
|
|
@ -73,6 +73,9 @@ hooks:
|
||||||
hostname: mongodb
|
hostname: mongodb
|
||||||
username: root
|
username: root
|
||||||
password: test
|
password: test
|
||||||
|
sqlite_databases:
|
||||||
|
- path: /tmp/sqlite_test.db
|
||||||
|
- name: sqlite_test
|
||||||
'''
|
'''
|
||||||
|
|
||||||
with open(config_path, 'w') as config_file:
|
with open(config_path, 'w') as config_file:
|
||||||
|
|
75
tests/unit/hooks/test_sqlite.py
Normal file
75
tests/unit/hooks/test_sqlite.py
Normal file
|
@ -0,0 +1,75 @@
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from flexmock import flexmock
|
||||||
|
|
||||||
|
from borgmatic.hooks import sqlite as module
|
||||||
|
|
||||||
|
|
||||||
|
def test_dump_databases_logs_and_skips_if_dump_already_exists():
|
||||||
|
databases = [{'path': '/path/to/database', 'name': 'database'}]
|
||||||
|
|
||||||
|
flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
|
||||||
|
flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
|
||||||
|
'/path/to/dump/database'
|
||||||
|
)
|
||||||
|
flexmock(module.os.path).should_receive('exists').and_return(True)
|
||||||
|
flexmock(logging).should_receive('info')
|
||||||
|
flexmock(logging).should_receive('warning')
|
||||||
|
|
||||||
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run = False) == []
|
||||||
|
|
||||||
|
def test_dump_databases_dumps_each_database():
|
||||||
|
databases = [
|
||||||
|
{'path': '/path/to/database1', 'name': 'database1'},
|
||||||
|
{'path': '/path/to/database2', 'name': 'database2'},
|
||||||
|
]
|
||||||
|
|
||||||
|
flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
|
||||||
|
flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
|
||||||
|
'/path/to/dump/database'
|
||||||
|
)
|
||||||
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
||||||
|
flexmock(logging).should_receive('info')
|
||||||
|
flexmock(logging).should_receive('warning')
|
||||||
|
flexmock(module).should_receive('dump.create_parent_directory_for_dump')
|
||||||
|
flexmock(module).should_receive('execute_command').and_return('process')
|
||||||
|
|
||||||
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run = False) == ['process', 'process']
|
||||||
|
|
||||||
|
def test_dump_databases_does_not_dump_if_dry_run():
|
||||||
|
databases = [{'path': '/path/to/database', 'name': 'database'}]
|
||||||
|
|
||||||
|
flexmock(module).should_receive('make_dump_path').and_return('/path/to/dump')
|
||||||
|
flexmock(module).should_receive('dump.make_database_dump_filename').and_return(
|
||||||
|
'/path/to/dump/database'
|
||||||
|
)
|
||||||
|
flexmock(module.os.path).should_receive('exists').and_return(False)
|
||||||
|
flexmock(logging).should_receive('info')
|
||||||
|
flexmock(logging).should_receive('warning')
|
||||||
|
flexmock(module).should_receive('dump.create_parent_directory_for_dump')
|
||||||
|
|
||||||
|
assert module.dump_databases(databases, 'test.yaml', {}, dry_run = True) == []
|
||||||
|
|
||||||
|
def test_restore_database_dump_restores_database():
|
||||||
|
database_config = [{'path': '/path/to/database', 'name': 'database'}]
|
||||||
|
extract_process = flexmock(stdout = flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').and_return('process')
|
||||||
|
|
||||||
|
module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = False, extract_process = extract_process)
|
||||||
|
|
||||||
|
def test_restore_database_dump_does_not_restore_database_if_dry_run():
|
||||||
|
database_config = [{'path': '/path/to/database', 'name': 'database'}]
|
||||||
|
extract_process = flexmock(stdout = flexmock())
|
||||||
|
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').never()
|
||||||
|
|
||||||
|
module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = True, extract_process = extract_process)
|
||||||
|
|
||||||
|
def test_restore_database_dump_raises_error_if_database_config_is_invalid():
|
||||||
|
database_config = []
|
||||||
|
extract_process = flexmock(stdout = flexmock())
|
||||||
|
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
module.restore_database_dump(database_config, 'test.yaml', {}, dry_run = False, extract_process = extract_process)
|
Loading…
Reference in a new issue