Skip auto-exclusion of special files when user explicitly sets read_special to true (#587).
This commit is contained in:
parent
902730df46
commit
a31ce337e9
3 changed files with 138 additions and 61 deletions
|
@ -332,7 +332,6 @@ def create_archive(
|
||||||
upload_rate_limit = storage_config.get('upload_rate_limit', None)
|
upload_rate_limit = storage_config.get('upload_rate_limit', None)
|
||||||
umask = storage_config.get('umask', None)
|
umask = storage_config.get('umask', None)
|
||||||
lock_wait = storage_config.get('lock_wait', None)
|
lock_wait = storage_config.get('lock_wait', None)
|
||||||
read_special = True if (location_config.get('read_special') or stream_processes) else False
|
|
||||||
files_cache = location_config.get('files_cache')
|
files_cache = location_config.get('files_cache')
|
||||||
archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
|
archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
|
||||||
extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
|
extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
|
||||||
|
@ -384,7 +383,7 @@ def create_archive(
|
||||||
+ atime_flags
|
+ atime_flags
|
||||||
+ (('--noctime',) if location_config.get('ctime') is False else ())
|
+ (('--noctime',) if location_config.get('ctime') is False else ())
|
||||||
+ (('--nobirthtime',) if location_config.get('birthtime') is False else ())
|
+ (('--nobirthtime',) if location_config.get('birthtime') is False else ())
|
||||||
+ (('--read-special',) if read_special else ())
|
+ (('--read-special',) if location_config.get('read_special') or stream_processes else ())
|
||||||
+ noflags_flags
|
+ noflags_flags
|
||||||
+ (('--files-cache', files_cache) if files_cache else ())
|
+ (('--files-cache', files_cache) if files_cache else ())
|
||||||
+ (('--remote-path', remote_path) if remote_path else ())
|
+ (('--remote-path', remote_path) if remote_path else ())
|
||||||
|
@ -410,8 +409,9 @@ def create_archive(
|
||||||
|
|
||||||
borg_environment = environment.make_environment(storage_config)
|
borg_environment = environment.make_environment(storage_config)
|
||||||
|
|
||||||
# If read_special is enabled, exclude files that might cause Borg to hang.
|
# If database hooks are enabled (as indicated by streaming processes), exclude files that might
|
||||||
if read_special:
|
# cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
|
||||||
|
if stream_processes and not location_config.get('read_special'):
|
||||||
logger.debug(f'{repository}: Collecting special file paths')
|
logger.debug(f'{repository}: Collecting special file paths')
|
||||||
special_file_paths = collect_special_file_paths(
|
special_file_paths = collect_special_file_paths(
|
||||||
create_command,
|
create_command,
|
||||||
|
|
|
@ -217,7 +217,11 @@ special files are excluded from backups (named pipes, block devices,
|
||||||
character devices, and sockets) to prevent hanging. Try a command like
|
character devices, and sockets) to prevent hanging. Try a command like
|
||||||
`find /your/source/path -type b -or -type c -or -type p -or -type s` to find
|
`find /your/source/path -type b -or -type c -or -type p -or -type s` to find
|
||||||
such files. Common directories to exclude are `/dev` and `/run`, but that may
|
such files. Common directories to exclude are `/dev` and `/run`, but that may
|
||||||
not be exhaustive.
|
not be exhaustive. <span class="minilink minilink-addedin">New in version
|
||||||
|
1.7.3</span> When database hooks are enabled, borgmatic automatically excludes
|
||||||
|
special files that may cause Borg to hang, so you no longer need to manually
|
||||||
|
exclude them. You can override/prevent this behavior by explicitly setting
|
||||||
|
`read_special` to true.
|
||||||
|
|
||||||
|
|
||||||
### Manual restoration
|
### Manual restoration
|
||||||
|
@ -273,3 +277,7 @@ Alternatively, if excluding special files is too onerous, you can create two
|
||||||
separate borgmatic configuration files—one for your source files and a
|
separate borgmatic configuration files—one for your source files and a
|
||||||
separate one for backing up databases. That way, the database `read_special`
|
separate one for backing up databases. That way, the database `read_special`
|
||||||
option will not be active when backing up special files.
|
option will not be active when backing up special files.
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">New in version 1.7.3</span> See
|
||||||
|
Limitations above about borgmatic's automatic exclusion of special files to
|
||||||
|
prevent Borg hangs.
|
||||||
|
|
|
@ -1147,62 +1147,6 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_create_archive_with_read_special_adds_special_files_to_excludes():
|
|
||||||
flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
|
|
||||||
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
|
|
||||||
flexmock(module).should_receive('map_directories_to_devices').and_return({})
|
|
||||||
flexmock(module).should_receive('expand_directories').and_return(())
|
|
||||||
flexmock(module).should_receive('pattern_root_directories').and_return([])
|
|
||||||
flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
|
|
||||||
flexmock(module).should_receive('expand_home_directories').and_return(()).and_return(
|
|
||||||
('special',)
|
|
||||||
)
|
|
||||||
flexmock(module).should_receive('write_pattern_file').and_return(None).and_return(
|
|
||||||
flexmock(name='/excludes')
|
|
||||||
)
|
|
||||||
flexmock(module.feature).should_receive('available').and_return(True)
|
|
||||||
flexmock(module).should_receive('ensure_files_readable')
|
|
||||||
flexmock(module).should_receive('make_pattern_flags').and_return(())
|
|
||||||
flexmock(module).should_receive('make_exclude_flags').and_return(()).and_return(
|
|
||||||
'--exclude-from', '/excludes'
|
|
||||||
)
|
|
||||||
flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
|
|
||||||
(f'repo::{DEFAULT_ARCHIVE_NAME}',)
|
|
||||||
)
|
|
||||||
flexmock(module.environment).should_receive('make_environment')
|
|
||||||
flexmock(module).should_receive('collect_special_file_paths').and_return(('special',))
|
|
||||||
create_command = ('borg', 'create', '--read-special') + REPO_ARCHIVE_WITH_PATHS
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
|
||||||
create_command + ('--dry-run', '--list'),
|
|
||||||
output_log_level=logging.INFO,
|
|
||||||
output_file=None,
|
|
||||||
borg_local_path='borg',
|
|
||||||
working_directory=None,
|
|
||||||
extra_environment=None,
|
|
||||||
)
|
|
||||||
flexmock(module).should_receive('execute_command').with_args(
|
|
||||||
create_command + ('--exclude-from', '/excludes'),
|
|
||||||
output_log_level=logging.INFO,
|
|
||||||
output_file=None,
|
|
||||||
borg_local_path='borg',
|
|
||||||
working_directory=None,
|
|
||||||
extra_environment=None,
|
|
||||||
)
|
|
||||||
|
|
||||||
module.create_archive(
|
|
||||||
dry_run=False,
|
|
||||||
repository='repo',
|
|
||||||
location_config={
|
|
||||||
'source_directories': ['foo', 'bar'],
|
|
||||||
'repositories': ['repo'],
|
|
||||||
'read_special': True,
|
|
||||||
'exclude_patterns': None,
|
|
||||||
},
|
|
||||||
storage_config={},
|
|
||||||
local_borg_version='1.2.3',
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
'option_name,option_value',
|
'option_name,option_value',
|
||||||
(('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),),
|
(('ctime', True), ('ctime', False), ('birthtime', True), ('birthtime', False),),
|
||||||
|
@ -1911,6 +1855,131 @@ def test_create_archive_with_stream_processes_ignores_read_special_false_and_log
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_archive_with_stream_processes_adds_special_files_to_excludes():
|
||||||
|
processes = flexmock()
|
||||||
|
flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
|
||||||
|
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
|
||||||
|
flexmock(module).should_receive('map_directories_to_devices').and_return({})
|
||||||
|
flexmock(module).should_receive('expand_directories').and_return(())
|
||||||
|
flexmock(module).should_receive('pattern_root_directories').and_return([])
|
||||||
|
flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
|
||||||
|
flexmock(module).should_receive('expand_home_directories').and_return(()).and_return(
|
||||||
|
('special',)
|
||||||
|
)
|
||||||
|
flexmock(module).should_receive('write_pattern_file').and_return(None).and_return(
|
||||||
|
flexmock(name='/excludes')
|
||||||
|
)
|
||||||
|
flexmock(module.feature).should_receive('available').and_return(True)
|
||||||
|
flexmock(module).should_receive('ensure_files_readable')
|
||||||
|
flexmock(module).should_receive('make_pattern_flags').and_return(())
|
||||||
|
flexmock(module).should_receive('make_exclude_flags').and_return(()).and_return(
|
||||||
|
'--exclude-from', '/excludes'
|
||||||
|
)
|
||||||
|
flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
|
||||||
|
(f'repo::{DEFAULT_ARCHIVE_NAME}',)
|
||||||
|
)
|
||||||
|
flexmock(module.environment).should_receive('make_environment')
|
||||||
|
flexmock(module).should_receive('collect_special_file_paths').and_return(('special',))
|
||||||
|
create_command = (
|
||||||
|
'borg',
|
||||||
|
'create',
|
||||||
|
'--one-file-system',
|
||||||
|
'--read-special',
|
||||||
|
) + REPO_ARCHIVE_WITH_PATHS
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
create_command + ('--dry-run', '--list'),
|
||||||
|
processes=processes,
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
output_file=None,
|
||||||
|
borg_local_path='borg',
|
||||||
|
working_directory=None,
|
||||||
|
extra_environment=None,
|
||||||
|
)
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
create_command + ('--exclude-from', '/excludes'),
|
||||||
|
processes=processes,
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
output_file=None,
|
||||||
|
borg_local_path='borg',
|
||||||
|
working_directory=None,
|
||||||
|
extra_environment=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
module.create_archive(
|
||||||
|
dry_run=False,
|
||||||
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
},
|
||||||
|
storage_config={},
|
||||||
|
local_borg_version='1.2.3',
|
||||||
|
stream_processes=processes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_archive_with_stream_processes_and_read_special_does_not_add_special_files_to_excludes():
|
||||||
|
processes = flexmock()
|
||||||
|
flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
|
||||||
|
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
|
||||||
|
flexmock(module).should_receive('map_directories_to_devices').and_return({})
|
||||||
|
flexmock(module).should_receive('expand_directories').and_return(())
|
||||||
|
flexmock(module).should_receive('pattern_root_directories').and_return([])
|
||||||
|
flexmock(module.os.path).should_receive('expanduser').and_raise(TypeError)
|
||||||
|
flexmock(module).should_receive('expand_home_directories').and_return(()).and_return(
|
||||||
|
('special',)
|
||||||
|
)
|
||||||
|
flexmock(module).should_receive('write_pattern_file').and_return(None)
|
||||||
|
flexmock(module.feature).should_receive('available').and_return(True)
|
||||||
|
flexmock(module).should_receive('ensure_files_readable')
|
||||||
|
flexmock(module).should_receive('make_pattern_flags').and_return(())
|
||||||
|
flexmock(module).should_receive('make_exclude_flags').and_return(())
|
||||||
|
flexmock(module.flags).should_receive('make_repository_archive_flags').and_return(
|
||||||
|
(f'repo::{DEFAULT_ARCHIVE_NAME}',)
|
||||||
|
)
|
||||||
|
flexmock(module.environment).should_receive('make_environment')
|
||||||
|
flexmock(module).should_receive('collect_special_file_paths').and_return(('special',))
|
||||||
|
create_command = (
|
||||||
|
'borg',
|
||||||
|
'create',
|
||||||
|
'--one-file-system',
|
||||||
|
'--read-special',
|
||||||
|
) + REPO_ARCHIVE_WITH_PATHS
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
create_command + ('--dry-run', '--list'),
|
||||||
|
processes=processes,
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
output_file=None,
|
||||||
|
borg_local_path='borg',
|
||||||
|
working_directory=None,
|
||||||
|
extra_environment=None,
|
||||||
|
)
|
||||||
|
flexmock(module).should_receive('execute_command_with_processes').with_args(
|
||||||
|
create_command,
|
||||||
|
processes=processes,
|
||||||
|
output_log_level=logging.INFO,
|
||||||
|
output_file=None,
|
||||||
|
borg_local_path='borg',
|
||||||
|
working_directory=None,
|
||||||
|
extra_environment=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
module.create_archive(
|
||||||
|
dry_run=False,
|
||||||
|
repository='repo',
|
||||||
|
location_config={
|
||||||
|
'source_directories': ['foo', 'bar'],
|
||||||
|
'repositories': ['repo'],
|
||||||
|
'exclude_patterns': None,
|
||||||
|
'read_special': True,
|
||||||
|
},
|
||||||
|
storage_config={},
|
||||||
|
local_borg_version='1.2.3',
|
||||||
|
stream_processes=processes,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_create_archive_with_json_calls_borg_with_json_parameter():
|
def test_create_archive_with_json_calls_borg_with_json_parameter():
|
||||||
flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
|
flexmock(module).should_receive('collect_borgmatic_source_directories').and_return([])
|
||||||
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
|
flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
|
||||||
|
|
Loading…
Reference in a new issue