From 16bebe983202a3bf15f250f585b3d906ae4e0f2e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 30 Oct 2014 22:34:03 -0700 Subject: [PATCH 001/189] Initial import. --- .hgignore | 3 +++ README | 51 +++++++++++++++++++++++++++++++++++++ atticmatic/__init__.py | 0 atticmatic/attic.py | 34 +++++++++++++++++++++++++ atticmatic/command.py | 38 ++++++++++++++++++++++++++++ atticmatic/config.py | 57 ++++++++++++++++++++++++++++++++++++++++++ sample/atticmatic.cron | 3 +++ sample/config | 12 +++++++++ sample/excludes | 3 +++ setup.py | 11 ++++++++ 10 files changed, 212 insertions(+) create mode 100644 .hgignore create mode 100644 README create mode 100644 atticmatic/__init__.py create mode 100644 atticmatic/attic.py create mode 100644 atticmatic/command.py create mode 100644 atticmatic/config.py create mode 100644 sample/atticmatic.cron create mode 100644 sample/config create mode 100644 sample/excludes create mode 100644 setup.py diff --git a/.hgignore b/.hgignore new file mode 100644 index 0000000..e91865a --- /dev/null +++ b/.hgignore @@ -0,0 +1,3 @@ +syntax: glob +*.pyc +*.egg-info diff --git a/README b/README new file mode 100644 index 0000000..5b61af7 --- /dev/null +++ b/README @@ -0,0 +1,51 @@ +Overview +-------- + +atticmatic is a simple Python wrapper script for the Attic backup software +that initiates a backup and prunes any old backups according to a retention +policy. The script supports specifying your settings in a declarative +configuration file rather than having to put them all on the command-line, and +handles common errors. + +Read more about Attic at https://attic-backup.org/ + + +Setup +----- + +To get up and running with Attic, follow the Attic Quick Start guide at +https://attic-backup.org/quickstart.html to create an Attic repository on a +local or remote host. + +If the repository is on a remote host, make sure that your local root user has +key-based ssh access to the desired user account on the remote host. + +To install atticmatic, run the following from the directory containing this +README: + + python setup.py install + +Then copy the following configuration files: + + sudo cp sample/atticmatic.cron /etc/init.d/atticmatic + sudo cp sample/config sample/excludes /etc/atticmatic/ + +Lastly, modify those files with your desired configuration. + + +Usage +----- + +You can run atticmatic and start a backup simply by invoking it without +arguments: + + atticmatic + +To get additional information about the progress of the backup, use the +verbose option: + + atticmattic --verbose + +If you'd like to see the available command-line arguments, view the help: + + atticmattic --help diff --git a/atticmatic/__init__.py b/atticmatic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/atticmatic/attic.py b/atticmatic/attic.py new file mode 100644 index 0000000..2274900 --- /dev/null +++ b/atticmatic/attic.py @@ -0,0 +1,34 @@ +from datetime import datetime + +import platform +import subprocess + + +def create_archive(excludes_filename, verbose, source_directories, repository): + sources = tuple(source_directories.split(' ')) + + command = ( + 'attic', 'create', + '--exclude-from', excludes_filename, + '{repo}::{hostname}-{timestamp}'.format( + repo=repository, + hostname=platform.node(), + timestamp=datetime.now().isoformat(), + ), + ) + sources + ( + ('--verbose', '--stats') if verbose else () + ) + + subprocess.check_call(command) + + +def prune_archives(repository, verbose, keep_daily, keep_weekly, keep_monthly): + command = ( + 'attic', 'prune', + repository, + '--keep-daily', str(keep_daily), + '--keep-weekly', str(keep_weekly), + '--keep-monthly', str(keep_monthly), + ) + (('--verbose',) if verbose else ()) + + subprocess.check_call(command) diff --git a/atticmatic/command.py b/atticmatic/command.py new file mode 100644 index 0000000..ebfdda3 --- /dev/null +++ b/atticmatic/command.py @@ -0,0 +1,38 @@ +from __future__ import print_function +from argparse import ArgumentParser +from subprocess import CalledProcessError +import sys + +from atticmatic.attic import create_archive, prune_archives +from atticmatic.config import parse_configuration + + +def main(): + parser = ArgumentParser() + parser.add_argument( + '--config', + dest='config_filename', + default='/etc/atticmatic/config', + help='Configuration filename', + ) + parser.add_argument( + '--excludes', + dest='excludes_filename', + default='/etc/atticmatic/excludes', + help='Excludes filename', + ) + parser.add_argument( + '--verbose', + action='store_true', + help='Display verbose progress information', + ) + args = parser.parse_args() + + try: + location_config, retention_config = parse_configuration(args.config_filename) + + create_archive(args.excludes_filename, args.verbose, *location_config) + prune_archives(location_config.repository, args.verbose, *retention_config) + except (ValueError, CalledProcessError), error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/atticmatic/config.py b/atticmatic/config.py new file mode 100644 index 0000000..7283ab8 --- /dev/null +++ b/atticmatic/config.py @@ -0,0 +1,57 @@ +from collections import namedtuple +from ConfigParser import SafeConfigParser + + +CONFIG_SECTION_LOCATION = 'location' +CONFIG_SECTION_RETENTION = 'retention' + +CONFIG_FORMAT = { + CONFIG_SECTION_LOCATION: ('source_directories', 'repository'), + CONFIG_SECTION_RETENTION: ('keep_daily', 'keep_weekly', 'keep_monthly'), +} + +LocationConfig = namedtuple('LocationConfig', CONFIG_FORMAT[CONFIG_SECTION_LOCATION]) +RetentionConfig = namedtuple('RetentionConfig', CONFIG_FORMAT[CONFIG_SECTION_RETENTION]) + + +def parse_configuration(config_filename): + ''' + Given a config filename of the expected format, return the parse configuration as a tuple of + (LocationConfig, RetentionConfig). Raise if the format is not as expected. + ''' + parser = SafeConfigParser() + parser.read((config_filename,)) + section_names = parser.sections() + expected_section_names = CONFIG_FORMAT.keys() + + if set(section_names) != set(expected_section_names): + raise ValueError( + 'Expected config sections {} but found sections: {}'.format( + ', '.join(expected_section_names), + ', '.join(section_names) + ) + ) + + for section_name in section_names: + option_names = parser.options(section_name) + expected_option_names = CONFIG_FORMAT[section_name] + + if set(option_names) != set(expected_option_names): + raise ValueError( + 'Expected options {} in config section {} but found options: {}'.format( + ', '.join(expected_option_names), + section_name, + ', '.join(option_names) + ) + ) + + return ( + LocationConfig(*( + parser.get(CONFIG_SECTION_LOCATION, option_name) + for option_name in CONFIG_FORMAT[CONFIG_SECTION_LOCATION] + )), + RetentionConfig(*( + parser.getint(CONFIG_SECTION_RETENTION, option_name) + for option_name in CONFIG_FORMAT[CONFIG_SECTION_RETENTION] + )) + ) diff --git a/sample/atticmatic.cron b/sample/atticmatic.cron new file mode 100644 index 0000000..a38e382 --- /dev/null +++ b/sample/atticmatic.cron @@ -0,0 +1,3 @@ +# You can drop this file into /etc/cron.d/ to run atticmatic nightly. + +0 3 * * * root /usr/local/bin/atticmatic diff --git a/sample/config b/sample/config new file mode 100644 index 0000000..66f508d --- /dev/null +++ b/sample/config @@ -0,0 +1,12 @@ +[location] +# Space-separated list of source directories to backup. +source_directories: /home /etc + +# Path to local or remote Attic repository. +repository: user@backupserver:sourcehostname.attic + +# Retention policy for how many backups to keep in each category. +[retention] +keep_daily: 7 +keep_weekly: 4 +keep_monthly: 6 diff --git a/sample/excludes b/sample/excludes new file mode 100644 index 0000000..7e81c88 --- /dev/null +++ b/sample/excludes @@ -0,0 +1,3 @@ +*.pyc +/home/*/.cache +/etc/ssl diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..5ebfca3 --- /dev/null +++ b/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup, find_packages + +setup( + name='atticmatic', + version='0.0.1', + description='A wrapper script for Attic backup software', + author='Dan Helfman', + author_email='witten@torsion.org', + packages=find_packages(), + entry_points={'console_scripts': ['atticmatic = atticmatic.command:main']}, +) From 84922c7232b26b0df1e51f4ecde629dc4076e264 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Nov 2014 17:46:04 -0700 Subject: [PATCH 002/189] Adding PATH necessary to find the attic binary. --- sample/atticmatic.cron | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample/atticmatic.cron b/sample/atticmatic.cron index a38e382..39bc6bc 100644 --- a/sample/atticmatic.cron +++ b/sample/atticmatic.cron @@ -1,3 +1,3 @@ # You can drop this file into /etc/cron.d/ to run atticmatic nightly. -0 3 * * * root /usr/local/bin/atticmatic +0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/atticmatic From db0f057b5413f63d3f4f1f8c27567b8ba9f6e8cf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Nov 2014 18:35:47 -0800 Subject: [PATCH 003/189] Adding contact info. --- README | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README b/README index 5b61af7..0378ec5 100644 --- a/README +++ b/README @@ -49,3 +49,9 @@ verbose option: If you'd like to see the available command-line arguments, view the help: atticmattic --help + + +Feedback +-------- + +Questions? Comments? Got a patch? Contact witten@torsion.org From e567158246fb0d572fd9df4e5e558d8521bc81d4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Nov 2014 21:57:44 -0800 Subject: [PATCH 004/189] Adding unit tests for config module. --- README | 12 ++++ atticmatic/tests/__init__.py | 0 atticmatic/tests/unit/__init__.py | 0 atticmatic/tests/unit/test_config.py | 83 ++++++++++++++++++++++++++++ setup.py | 4 ++ 5 files changed, 99 insertions(+) create mode 100644 atticmatic/tests/__init__.py create mode 100644 atticmatic/tests/unit/__init__.py create mode 100644 atticmatic/tests/unit/test_config.py diff --git a/README b/README index 0378ec5..604a6b4 100644 --- a/README +++ b/README @@ -51,6 +51,18 @@ If you'd like to see the available command-line arguments, view the help: atticmattic --help +Running tests +------------- + +To install test-specific dependencies, first run: + + python setup.py test + +To actually run tests, run: + + nosetests --detailed-errors + + Feedback -------- diff --git a/atticmatic/tests/__init__.py b/atticmatic/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/atticmatic/tests/unit/__init__.py b/atticmatic/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py new file mode 100644 index 0000000..93cf509 --- /dev/null +++ b/atticmatic/tests/unit/test_config.py @@ -0,0 +1,83 @@ +from flexmock import flexmock +from nose.tools import assert_raises + +from atticmatic import config as module + + +def insert_mock_parser(section_names): + parser = flexmock() + parser.should_receive('read') + parser.should_receive('sections').and_return(section_names) + flexmock(module).SafeConfigParser = parser + + return parser + + +def test_parse_configuration_should_return_config_data(): + section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION) + parser = insert_mock_parser(section_names) + + for section_name in section_names: + parser.should_receive('options').with_args(section_name).and_return( + module.CONFIG_FORMAT[section_name], + ) + + expected_config = ( + module.LocationConfig(flexmock(), flexmock()), + module.RetentionConfig(flexmock(), flexmock(), flexmock()), + ) + sections = ( + (module.CONFIG_SECTION_LOCATION, expected_config[0], 'get'), + (module.CONFIG_SECTION_RETENTION, expected_config[1], 'getint'), + ) + + for section_name, section_config, method_name in sections: + for index, option_name in enumerate(module.CONFIG_FORMAT[section_name]): + ( + parser.should_receive(method_name).with_args(section_name, option_name) + .and_return(section_config[index]) + ) + + config = module.parse_configuration(flexmock()) + + assert config == expected_config + + +def test_parse_configuration_with_missing_section_should_raise(): + insert_mock_parser((module.CONFIG_SECTION_LOCATION,)) + + with assert_raises(ValueError): + module.parse_configuration(flexmock()) + + +def test_parse_configuration_with_extra_section_should_raise(): + insert_mock_parser((module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION, 'extra')) + + with assert_raises(ValueError): + module.parse_configuration(flexmock()) + + +def test_parse_configuration_with_missing_option_should_raise(): + section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION) + parser = insert_mock_parser(section_names) + + for section_name in section_names: + parser.should_receive('options').with_args(section_name).and_return( + module.CONFIG_FORMAT[section_name][:-1], + ) + + with assert_raises(ValueError): + module.parse_configuration(flexmock()) + + +def test_parse_configuration_with_extra_option_should_raise(): + section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION) + parser = insert_mock_parser(section_names) + + for section_name in section_names: + parser.should_receive('options').with_args(section_name).and_return( + module.CONFIG_FORMAT[section_name] + ('extra',), + ) + + with assert_raises(ValueError): + module.parse_configuration(flexmock()) diff --git a/setup.py b/setup.py index 5ebfca3..209db30 100644 --- a/setup.py +++ b/setup.py @@ -8,4 +8,8 @@ setup( author_email='witten@torsion.org', packages=find_packages(), entry_points={'console_scripts': ['atticmatic = atticmatic.command:main']}, + tests_require=( + 'flexmock', + 'nose', + ) ) From d1825097714c099a335609d7a3bf7c1afb7eace0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Nov 2014 22:19:34 -0800 Subject: [PATCH 005/189] Unit tests for attic invocation code. --- atticmatic/tests/unit/test_attic.py | 86 +++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 atticmatic/tests/unit/test_attic.py diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py new file mode 100644 index 0000000..9cce587 --- /dev/null +++ b/atticmatic/tests/unit/test_attic.py @@ -0,0 +1,86 @@ +from flexmock import flexmock + +from atticmatic import attic as module + + +def insert_subprocess_mock(check_call_command): + subprocess = flexmock() + subprocess.should_receive('check_call').with_args(check_call_command).once() + flexmock(module).subprocess = subprocess + + +def insert_platform_mock(): + flexmock(module).platform = flexmock().should_receive('node').and_return('host').mock + + +def insert_datetime_mock(): + flexmock(module).datetime = flexmock().should_receive('now').and_return( + flexmock().should_receive('isoformat').and_return('now').mock + ).mock + + +def test_create_archive_should_call_attic_with_parameters(): + insert_subprocess_mock( + ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'), + ) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + excludes_filename='excludes', + verbose=False, + source_directories='foo bar', + repository='repo', + ) + + +def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters(): + insert_subprocess_mock( + ( + 'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar', + '--verbose', '--stats', + ), + ) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + excludes_filename='excludes', + verbose=True, + source_directories='foo bar', + repository='repo', + ) + + +def test_prune_archives_should_call_attic_with_parameters(): + insert_subprocess_mock( + ( + 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', + '3', + ), + ) + + module.prune_archives( + repository='repo', + verbose=False, + keep_daily=1, + keep_weekly=2, + keep_monthly=3 + ) + + +def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters(): + insert_subprocess_mock( + ( + 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', + '3', '--verbose', + ), + ) + + module.prune_archives( + repository='repo', + verbose=True, + keep_daily=1, + keep_weekly=2, + keep_monthly=3 + ) From 42d9e2bfd868eb2c79906cf6500aef94948dcc90 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 18 Nov 2014 18:22:51 -0800 Subject: [PATCH 006/189] Adding GPL v3 license. --- LICENSE | 675 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 675 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6b156fe --- /dev/null +++ b/LICENSE @@ -0,0 +1,675 @@ +GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + {one line to give the program's name and a brief idea of what it does.} + Copyright (C) {year} {name of author} + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + {project} Copyright (C) {year} {fullname} + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + From cf4c2622262267c987bdfdf2802ea41221ae8e1f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 18 Nov 2014 18:32:16 -0800 Subject: [PATCH 007/189] Note about hosting arrangement. --- README | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README b/README index 604a6b4..9d4e7cb 100644 --- a/README +++ b/README @@ -9,6 +9,9 @@ handles common errors. Read more about Attic at https://attic-backup.org/ +atticmatic is hosted at http://torsion.org/hg/atticmatic/ and is mirrored on +GitHub and BitBucket for convenience. + Setup ----- From 200a1bd63e9e3ad9b6b1e406ff17a2f335aa9df7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 25 Nov 2014 16:01:59 -0800 Subject: [PATCH 008/189] Updating README with clarifications and examples. --- README | 24 ++++++++++++++++++++++-- sample/config | 2 +- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/README b/README index 9d4e7cb..3630199 100644 --- a/README +++ b/README @@ -7,6 +7,24 @@ policy. The script supports specifying your settings in a declarative configuration file rather than having to put them all on the command-line, and handles common errors. +Here's an example config file: + + [location] + # Space-separated list of source directories to backup. + source_directories: /home /etc + + # Path to local or remote Attic repository. + repository: user@backupserver:sourcehostname.attic + + [retention] + # Retention policy for how many backups to keep in each category. + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 6 + +Additionally, exclude patterns can be specified in a separate excludes config +file, one pattern per line. + Read more about Attic at https://attic-backup.org/ atticmatic is hosted at http://torsion.org/hg/atticmatic/ and is mirrored on @@ -44,8 +62,10 @@ arguments: atticmatic -To get additional information about the progress of the backup, use the -verbose option: +This will also prune any old backups as per the configured retention policy. +By default, the backup will proceed silently except in the case of errors. But +if you'd like to to get additional information about the progress of the +backup as it proceeds, use the verbose option instead: atticmattic --verbose diff --git a/sample/config b/sample/config index 66f508d..3d7a111 100644 --- a/sample/config +++ b/sample/config @@ -5,8 +5,8 @@ source_directories: /home /etc # Path to local or remote Attic repository. repository: user@backupserver:sourcehostname.attic -# Retention policy for how many backups to keep in each category. [retention] +# Retention policy for how many backups to keep in each category. keep_daily: 7 keep_weekly: 4 keep_monthly: 6 From 704b97a636e7a98901ea12aee00b42a122cce3c1 Mon Sep 17 00:00:00 2001 From: Henning Schroder Date: Wed, 26 Nov 2014 13:04:14 +0100 Subject: [PATCH 009/189] fixed README: copy cronjob to /etc/cron.d instead of /etc/init.d (like comment in sample/atticmatic.cron correctly explains) --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 3630199..9a2a628 100644 --- a/README +++ b/README @@ -48,7 +48,7 @@ README: Then copy the following configuration files: - sudo cp sample/atticmatic.cron /etc/init.d/atticmatic + sudo cp sample/atticmatic.cron /etc/cron.d/atticmatic sudo cp sample/config sample/excludes /etc/atticmatic/ Lastly, modify those files with your desired configuration. From 10a449fe1af6fd6b1f977c4e22aed21c9e554251 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 26 Nov 2014 20:15:21 -0800 Subject: [PATCH 010/189] Adding note about making etc configuration directory before copying a file to it. --- README | 1 + 1 file changed, 1 insertion(+) diff --git a/README b/README index 9a2a628..ebdd062 100644 --- a/README +++ b/README @@ -49,6 +49,7 @@ README: Then copy the following configuration files: sudo cp sample/atticmatic.cron /etc/cron.d/atticmatic + sudo mkdir /etc/atticmatic/ sudo cp sample/config sample/excludes /etc/atticmatic/ Lastly, modify those files with your desired configuration. From 5472424d5a3e1501584836d88f928c4cee68c3de Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Nov 2014 09:29:31 -0800 Subject: [PATCH 011/189] Playing nicely with markdown. --- README | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/README b/README index ebdd062..b30ebc1 100644 --- a/README +++ b/README @@ -1,5 +1,4 @@ -Overview --------- +## Overview atticmatic is a simple Python wrapper script for the Attic backup software that initiates a backup and prunes any old backups according to a retention @@ -31,8 +30,7 @@ atticmatic is hosted at http://torsion.org/hg/atticmatic/ and is mirrored on GitHub and BitBucket for convenience. -Setup ------ +## Setup To get up and running with Attic, follow the Attic Quick Start guide at https://attic-backup.org/quickstart.html to create an Attic repository on a @@ -55,8 +53,7 @@ Then copy the following configuration files: Lastly, modify those files with your desired configuration. -Usage ------ +## Usage You can run atticmatic and start a backup simply by invoking it without arguments: @@ -75,8 +72,7 @@ If you'd like to see the available command-line arguments, view the help: atticmattic --help -Running tests -------------- +## Running tests To install test-specific dependencies, first run: @@ -87,7 +83,6 @@ To actually run tests, run: nosetests --detailed-errors -Feedback --------- +## Feedback Questions? Comments? Got a patch? Contact witten@torsion.org From f862eda7d6db61007bd1c5e77a00364218ab45de Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Nov 2014 09:34:13 -0800 Subject: [PATCH 012/189] Renaming README to indicate markdown. --- README => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README => README.md (100%) diff --git a/README b/README.md similarity index 100% rename from README rename to README.md From 814770c2a914c7b5389150bd67edfcf522eb8049 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 1 Dec 2014 19:49:25 -0800 Subject: [PATCH 013/189] Markdown metadata and link formatting updates. --- README.md | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b30ebc1..1b0f525 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ +title: Atticmatic +date: + ## Overview -atticmatic is a simple Python wrapper script for the Attic backup software -that initiates a backup and prunes any old backups according to a retention -policy. The script supports specifying your settings in a declarative -configuration file rather than having to put them all on the command-line, and -handles common errors. +atticmatic is a simple Python wrapper script for the [Attic backup +software](https://attic-backup.org/) that initiates a backup and prunes any +old backups according to a retention policy. The script supports specifying +your settings in a declarative configuration file rather than having to put +them all on the command-line, and handles common errors. Here's an example config file: @@ -24,25 +27,23 @@ Here's an example config file: Additionally, exclude patterns can be specified in a separate excludes config file, one pattern per line. -Read more about Attic at https://attic-backup.org/ - -atticmatic is hosted at http://torsion.org/hg/atticmatic/ and is mirrored on -GitHub and BitBucket for convenience. +atticmatic is hosted at and is mirrored on +[GitHub](https://github.com/witten/atticmatic) and +[BitBucket](https://bitbucket.org/dhelfman/atticmatic) for convenience. ## Setup -To get up and running with Attic, follow the Attic Quick Start guide at -https://attic-backup.org/quickstart.html to create an Attic repository on a -local or remote host. +To get up and running with Attic, follow the [Attic Quick +Start](https://attic-backup.org/quickstart.html) guide to create an Attic +repository on a local or remote host. If the repository is on a remote host, make sure that your local root user has key-based ssh access to the desired user account on the remote host. -To install atticmatic, run the following from the directory containing this -README: +To install atticmatic, run the following command to download and install it: - python setup.py install + sudo pip install hg+https://torsion.org/hg/atticmatic Then copy the following configuration files: @@ -85,4 +86,4 @@ To actually run tests, run: ## Feedback -Questions? Comments? Got a patch? Contact witten@torsion.org +Questions? Comments? Got a patch? Contact . From 8a4167b7a3653b1910573d7132c307f82845b23a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 1 Dec 2014 20:15:21 -0800 Subject: [PATCH 014/189] Saving README when rendered such that it can be served easily. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1b0f525..69aec5a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ title: Atticmatic date: +save_as: atticmatic/index.html ## Overview From 65c837c828595d359cd8cf8285c5606511ef40ac Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 1 Dec 2014 20:22:49 -0800 Subject: [PATCH 015/189] Mentioning source code location explicitly. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 69aec5a..9e91077 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ Here's an example config file: Additionally, exclude patterns can be specified in a separate excludes config file, one pattern per line. -atticmatic is hosted at and is mirrored on +atticmatic is hosted at with [source code +available](https://torsion.org/hg/atticmatic). It's also mirrored on [GitHub](https://github.com/witten/atticmatic) and [BitBucket](https://bitbucket.org/dhelfman/atticmatic) for convenience. From 45c65412668170c10463717ebca72b905caecc6e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 1 Dec 2014 20:23:29 -0800 Subject: [PATCH 016/189] Python 3 compatible exceptions. --- atticmatic/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atticmatic/command.py b/atticmatic/command.py index ebfdda3..1466c30 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -33,6 +33,6 @@ def main(): create_archive(args.excludes_filename, args.verbose, *location_config) prune_archives(location_config.repository, args.verbose, *retention_config) - except (ValueError, CalledProcessError), error: + except (ValueError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) From 69971cd7e251246f1010e31eca8e84458240dac9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 1 Dec 2014 20:26:19 -0800 Subject: [PATCH 017/189] Python 3 ConfigParser compatibility. --- atticmatic/config.py | 4 ++-- atticmatic/tests/unit/test_config.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/atticmatic/config.py b/atticmatic/config.py index 7283ab8..c11b784 100644 --- a/atticmatic/config.py +++ b/atticmatic/config.py @@ -1,5 +1,5 @@ from collections import namedtuple -from ConfigParser import SafeConfigParser +from ConfigParser import ConfigParser CONFIG_SECTION_LOCATION = 'location' @@ -19,7 +19,7 @@ def parse_configuration(config_filename): Given a config filename of the expected format, return the parse configuration as a tuple of (LocationConfig, RetentionConfig). Raise if the format is not as expected. ''' - parser = SafeConfigParser() + parser = ConfigParser() parser.read((config_filename,)) section_names = parser.sections() expected_section_names = CONFIG_FORMAT.keys() diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py index 93cf509..818d3db 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/atticmatic/tests/unit/test_config.py @@ -8,7 +8,7 @@ def insert_mock_parser(section_names): parser = flexmock() parser.should_receive('read') parser.should_receive('sections').and_return(section_names) - flexmock(module).SafeConfigParser = parser + flexmock(module).ConfigParser = parser return parser From d0eae195565b60cbfe0194d96e08bc7e60c9b4b6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 1 Dec 2014 20:30:07 -0800 Subject: [PATCH 018/189] Adding authors/contributors file. --- AUTHORS | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 AUTHORS diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..2a2f28f --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ +Dan Helfman : Main developer + +Alexander Görtz: Python 3 compatibility +Henning Schroeder: Copy editing From 126bb279cde133c65eda364da7dd660c22db926f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 1 Dec 2014 20:36:43 -0800 Subject: [PATCH 019/189] Expanding description. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 209db30..d23ba77 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages setup( name='atticmatic', version='0.0.1', - description='A wrapper script for Attic backup software', + description='A wrapper script for Attic backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', packages=find_packages(), From d46e37095000418d234146810e31ff57f25498dd Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 1 Dec 2014 22:14:35 -0800 Subject: [PATCH 020/189] Fixing configparser import for Python 3. --- atticmatic/config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/atticmatic/config.py b/atticmatic/config.py index c11b784..20caa34 100644 --- a/atticmatic/config.py +++ b/atticmatic/config.py @@ -1,5 +1,11 @@ from collections import namedtuple -from ConfigParser import ConfigParser + +try: + # Python 2 + from ConfigParser import ConfigParser +except ImportError: + # Python 3 + from configparser import ConfigParser CONFIG_SECTION_LOCATION = 'location' From 626dd66254b83d8b78b05926e1966961ae54a64f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 1 Dec 2014 22:35:25 -0800 Subject: [PATCH 021/189] Preventing ConfigParser from swallowing file read IOErrors, so that the user gets a more useful message. --- atticmatic/command.py | 13 ++++++++++--- atticmatic/config.py | 6 ++++-- atticmatic/tests/unit/test_config.py | 3 ++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/atticmatic/command.py b/atticmatic/command.py index 1466c30..7ea236b 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -7,7 +7,10 @@ from atticmatic.attic import create_archive, prune_archives from atticmatic.config import parse_configuration -def main(): +def parse_arguments(): + ''' + Parse the command-line arguments from sys.argv and return them as an ArgumentParser instance. + ''' parser = ArgumentParser() parser.add_argument( '--config', @@ -26,13 +29,17 @@ def main(): action='store_true', help='Display verbose progress information', ) - args = parser.parse_args() + return parser.parse_args() + + +def main(): try: + args = parse_arguments() location_config, retention_config = parse_configuration(args.config_filename) create_archive(args.excludes_filename, args.verbose, *location_config) prune_archives(location_config.repository, args.verbose, *retention_config) - except (ValueError, CalledProcessError) as error: + except (ValueError, IOError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/atticmatic/config.py b/atticmatic/config.py index 20caa34..ac8cad1 100644 --- a/atticmatic/config.py +++ b/atticmatic/config.py @@ -23,10 +23,12 @@ RetentionConfig = namedtuple('RetentionConfig', CONFIG_FORMAT[CONFIG_SECTION_RET def parse_configuration(config_filename): ''' Given a config filename of the expected format, return the parse configuration as a tuple of - (LocationConfig, RetentionConfig). Raise if the format is not as expected. + (LocationConfig, RetentionConfig). + + Raise IOError if the file cannot be read, or ValueError if the format is not as expected. ''' parser = ConfigParser() - parser.read((config_filename,)) + parser.readfp(open(config_filename)) section_names = parser.sections() expected_section_names = CONFIG_FORMAT.keys() diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py index 818d3db..f393533 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/atticmatic/tests/unit/test_config.py @@ -6,8 +6,9 @@ from atticmatic import config as module def insert_mock_parser(section_names): parser = flexmock() - parser.should_receive('read') + parser.should_receive('readfp') parser.should_receive('sections').and_return(section_names) + flexmock(module).open = lambda filename: None flexmock(module).ConfigParser = parser return parser From 965dd1aabe0f56c3604b2489354cd131b2021fad Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 1 Dec 2014 22:39:11 -0800 Subject: [PATCH 022/189] Adding sudo to installation of test dependencies, for consistency with installation of main dependencies. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e91077..b02984b 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ If you'd like to see the available command-line arguments, view the help: To install test-specific dependencies, first run: - python setup.py test + sudo python setup.py test To actually run tests, run: From b94c106a36e5493884bc388b7e378b70b7cd3c6c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 1 Dec 2014 22:47:51 -0800 Subject: [PATCH 023/189] For convenience, adding some short-form arguments in addition to the long-form arguments. --- atticmatic/command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/atticmatic/command.py b/atticmatic/command.py index 7ea236b..e9e8c9b 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -13,7 +13,7 @@ def parse_arguments(): ''' parser = ArgumentParser() parser.add_argument( - '--config', + '-c', '--config', dest='config_filename', default='/etc/atticmatic/config', help='Configuration filename', @@ -25,7 +25,7 @@ def parse_arguments(): help='Excludes filename', ) parser.add_argument( - '--verbose', + '-v', '--verbose', action='store_true', help='Display verbose progress information', ) From 056ed7184b60f0df457927cbcde08c0ab7940198 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 6 Dec 2014 18:35:20 -0800 Subject: [PATCH 024/189] Configuration support for additional attic prune flags: keep_within, keep_hourly, keep_yearly, and prefix. --- .hgignore | 3 +- NEWS | 8 + atticmatic/attic.py | 39 ++++- atticmatic/command.py | 4 +- atticmatic/config.py | 147 ++++++++++++------ atticmatic/tests/unit/test_attic.py | 41 ++++- atticmatic/tests/unit/test_config.py | 216 ++++++++++++++++++--------- sample/config | 5 + setup.py | 2 +- 9 files changed, 341 insertions(+), 124 deletions(-) create mode 100644 NEWS diff --git a/.hgignore b/.hgignore index e91865a..abab077 100644 --- a/.hgignore +++ b/.hgignore @@ -1,3 +1,4 @@ syntax: glob -*.pyc *.egg-info +*.pyc +*.swp diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..e9e6290 --- /dev/null +++ b/NEWS @@ -0,0 +1,8 @@ +0.0.2 + + * Configuration support for additional attic prune flags: keep_within, keep_hourly, keep_yearly, + and prefix. + +0.0.1 + + * Initial release. diff --git a/atticmatic/attic.py b/atticmatic/attic.py index 2274900..c884210 100644 --- a/atticmatic/attic.py +++ b/atticmatic/attic.py @@ -5,6 +5,10 @@ import subprocess def create_archive(excludes_filename, verbose, source_directories, repository): + ''' + Given an excludes filename, a vebosity flag, a space-separated list of source directories, and + a local or remote repository path, create an attic archive. + ''' sources = tuple(source_directories.split(' ')) command = ( @@ -22,13 +26,40 @@ def create_archive(excludes_filename, verbose, source_directories, repository): subprocess.check_call(command) -def prune_archives(repository, verbose, keep_daily, keep_weekly, keep_monthly): +def make_prune_flags(retention_config): + ''' + Given a retention config dict mapping from option name to value, tranform it into an iterable of + command-line name-value flag pairs. + + For example, given a retention config of: + + {'keep_weekly': 4, 'keep_monthly': 6} + + This will be returned as an iterable of: + + ( + ('--keep-weekly', '4'), + ('--keep-monthly', '6'), + ) + ''' + return ( + ('--' + option_name.replace('_', '-'), str(retention_config[option_name])) + for option_name, value in retention_config.items() + ) + + +def prune_archives(verbose, repository, retention_config): + ''' + Given a verbosity flag, a local or remote repository path, and a retention config dict, prune + attic archives according the the retention policy specified in that configuration. + ''' command = ( 'attic', 'prune', repository, - '--keep-daily', str(keep_daily), - '--keep-weekly', str(keep_weekly), - '--keep-monthly', str(keep_monthly), + ) + tuple( + element + for pair in make_prune_flags(retention_config) + for element in pair ) + (('--verbose',) if verbose else ()) subprocess.check_call(command) diff --git a/atticmatic/command.py b/atticmatic/command.py index e9e8c9b..d47f63a 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -38,8 +38,8 @@ def main(): args = parse_arguments() location_config, retention_config = parse_configuration(args.config_filename) - create_archive(args.excludes_filename, args.verbose, *location_config) - prune_archives(location_config.repository, args.verbose, *retention_config) + create_archive(args.excludes_filename, args.verbose, **location_config) + prune_archives(args.verbose, location_config['repository'], retention_config) except (ValueError, IOError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/atticmatic/config.py b/atticmatic/config.py index ac8cad1..f953860 100644 --- a/atticmatic/config.py +++ b/atticmatic/config.py @@ -1,4 +1,4 @@ -from collections import namedtuple +from collections import OrderedDict, namedtuple try: # Python 2 @@ -8,58 +8,121 @@ except ImportError: from configparser import ConfigParser -CONFIG_SECTION_LOCATION = 'location' -CONFIG_SECTION_RETENTION = 'retention' +Section_format = namedtuple('Section_format', ('name', 'options')) +Config_option = namedtuple('Config_option', ('name', 'value_type', 'required')) -CONFIG_FORMAT = { - CONFIG_SECTION_LOCATION: ('source_directories', 'repository'), - CONFIG_SECTION_RETENTION: ('keep_daily', 'keep_weekly', 'keep_monthly'), -} -LocationConfig = namedtuple('LocationConfig', CONFIG_FORMAT[CONFIG_SECTION_LOCATION]) -RetentionConfig = namedtuple('RetentionConfig', CONFIG_FORMAT[CONFIG_SECTION_RETENTION]) +def option(name, value_type=str, required=True): + ''' + Given a config file option name, an expected type for its value, and whether it's required, + return a Config_option capturing that information. + ''' + return Config_option(name, value_type, required) + + +CONFIG_FORMAT = ( + Section_format( + 'location', + ( + option('source_directories'), + option('repository'), + ), + ), + Section_format( + 'retention', + ( + option('keep_within', required=False), + option('keep_hourly', int, required=False), + option('keep_daily', int, required=False), + option('keep_weekly', int, required=False), + option('keep_monthly', int, required=False), + option('keep_yearly', int, required=False), + option('prefix', required=False), + ), + ) +) + + +def validate_configuration_format(parser, config_format): + ''' + Given an open ConfigParser and an expected config file format, validate that the parsed + configuration file has the expected sections, that any required options are present in those + sections, and that there aren't any unexpected options. + + Raise ValueError if anything is awry. + ''' + section_names = parser.sections() + required_section_names = tuple(section.name for section in config_format) + + if set(section_names) != set(required_section_names): + raise ValueError( + 'Expected config sections {} but found sections: {}'.format( + ', '.join(required_section_names), + ', '.join(section_names) + ) + ) + + for section_format in config_format: + option_names = parser.options(section_format.name) + expected_options = section_format.options + + unexpected_option_names = set(option_names) - set(option.name for option in expected_options) + + if unexpected_option_names: + raise ValueError( + 'Unexpected options found in config section {}: {}'.format( + section_format.name, + ', '.join(sorted(unexpected_option_names)), + ) + ) + + missing_option_names = tuple( + option.name for option in expected_options if option.required + if option.name not in option_names + ) + + if missing_option_names: + raise ValueError( + 'Required options missing from config section {}: {}'.format( + section_format.name, + ', '.join(missing_option_names) + ) + ) + + +def parse_section_options(parser, section_format): + ''' + Given an open ConfigParser and an expected section format, return the option values from that + section as a dict mapping from option name to value. Omit those options that are not present in + the parsed options. + + Raise ValueError if any option values cannot be coerced to the expected Python data type. + ''' + type_getter = { + str: parser.get, + int: parser.getint, + } + + return OrderedDict( + (option.name, type_getter[option.value_type](section_format.name, option.name)) + for option in section_format.options + if parser.has_option(section_format.name, option.name) + ) def parse_configuration(config_filename): ''' - Given a config filename of the expected format, return the parse configuration as a tuple of - (LocationConfig, RetentionConfig). + Given a config filename of the expected format, return the parsed configuration as a tuple of + (location config, retention config) where each config is a dict of that section's options. Raise IOError if the file cannot be read, or ValueError if the format is not as expected. ''' parser = ConfigParser() parser.readfp(open(config_filename)) - section_names = parser.sections() - expected_section_names = CONFIG_FORMAT.keys() - if set(section_names) != set(expected_section_names): - raise ValueError( - 'Expected config sections {} but found sections: {}'.format( - ', '.join(expected_section_names), - ', '.join(section_names) - ) - ) + validate_configuration_format(parser, CONFIG_FORMAT) - for section_name in section_names: - option_names = parser.options(section_name) - expected_option_names = CONFIG_FORMAT[section_name] - - if set(option_names) != set(expected_option_names): - raise ValueError( - 'Expected options {} in config section {} but found options: {}'.format( - ', '.join(expected_option_names), - section_name, - ', '.join(option_names) - ) - ) - - return ( - LocationConfig(*( - parser.get(CONFIG_SECTION_LOCATION, option_name) - for option_name in CONFIG_FORMAT[CONFIG_SECTION_LOCATION] - )), - RetentionConfig(*( - parser.getint(CONFIG_SECTION_RETENTION, option_name) - for option_name in CONFIG_FORMAT[CONFIG_SECTION_RETENTION] - )) + return tuple( + parse_section_options(parser, section_format) + for section_format in CONFIG_FORMAT ) diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py index 9cce587..44b38bb 100644 --- a/atticmatic/tests/unit/test_attic.py +++ b/atticmatic/tests/unit/test_attic.py @@ -1,3 +1,5 @@ +from collections import OrderedDict + from flexmock import flexmock from atticmatic import attic as module @@ -52,7 +54,32 @@ def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters() ) +BASE_PRUNE_FLAGS = ( + ('--keep-daily', '1'), + ('--keep-weekly', '2'), + ('--keep-monthly', '3'), +) + + +def test_make_prune_flags_should_return_flags_from_config(): + retention_config = OrderedDict( + ( + ('keep_daily', 1), + ('keep_weekly', 2), + ('keep_monthly', 3), + ) + ) + + result = module.make_prune_flags(retention_config) + + assert tuple(result) == BASE_PRUNE_FLAGS + + def test_prune_archives_should_call_attic_with_parameters(): + retention_config = flexmock() + flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return( + BASE_PRUNE_FLAGS, + ) insert_subprocess_mock( ( 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', @@ -61,15 +88,17 @@ def test_prune_archives_should_call_attic_with_parameters(): ) module.prune_archives( - repository='repo', verbose=False, - keep_daily=1, - keep_weekly=2, - keep_monthly=3 + repository='repo', + retention_config=retention_config, ) def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters(): + retention_config = flexmock() + flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return( + BASE_PRUNE_FLAGS, + ) insert_subprocess_mock( ( 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', @@ -80,7 +109,5 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters() module.prune_archives( repository='repo', verbose=True, - keep_daily=1, - keep_weekly=2, - keep_monthly=3 + retention_config=retention_config, ) diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py index f393533..0576dc8 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/atticmatic/tests/unit/test_config.py @@ -1,84 +1,166 @@ +from collections import OrderedDict + from flexmock import flexmock from nose.tools import assert_raises from atticmatic import config as module -def insert_mock_parser(section_names): +def test_option_should_create_config_option(): + option = module.option('name', bool, required=False) + + assert option == module.Config_option('name', bool, False) + + +def test_option_should_create_config_option_with_defaults(): + option = module.option('name') + + assert option == module.Config_option('name', str, True) + + +def test_validate_configuration_format_with_valid_config_should_not_raise(): + parser = flexmock() + parser.should_receive('sections').and_return(('section', 'other')) + parser.should_receive('options').with_args('section').and_return(('stuff',)) + parser.should_receive('options').with_args('other').and_return(('such',)) + config_format = ( + module.Section_format( + 'section', + options=( + module.Config_option('stuff', str, required=True), + ), + ), + module.Section_format( + 'other', + options=( + module.Config_option('such', str, required=True), + ), + ), + ) + + module.validate_configuration_format(parser, config_format) + + +def test_validate_configuration_format_with_missing_section_should_raise(): + parser = flexmock() + parser.should_receive('sections').and_return(('section',)) + config_format = ( + module.Section_format('section', options=()), + module.Section_format('missing', options=()), + ) + + with assert_raises(ValueError): + module.validate_configuration_format(parser, config_format) + + +def test_validate_configuration_format_with_extra_section_should_raise(): + parser = flexmock() + parser.should_receive('sections').and_return(('section', 'extra')) + config_format = ( + module.Section_format('section', options=()), + ) + + with assert_raises(ValueError): + module.validate_configuration_format(parser, config_format) + + +def test_validate_configuration_format_with_missing_required_option_should_raise(): + parser = flexmock() + parser.should_receive('sections').and_return(('section',)) + parser.should_receive('options').with_args('section').and_return(('option',)) + config_format = ( + module.Section_format( + 'section', + options=( + module.Config_option('option', str, required=True), + module.Config_option('missing', str, required=True), + ), + ), + ) + + with assert_raises(ValueError): + module.validate_configuration_format(parser, config_format) + + +def test_validate_configuration_format_with_missing_optional_option_should_not_raise(): + parser = flexmock() + parser.should_receive('sections').and_return(('section',)) + parser.should_receive('options').with_args('section').and_return(('option',)) + config_format = ( + module.Section_format( + 'section', + options=( + module.Config_option('option', str, required=True), + module.Config_option('missing', str, required=False), + ), + ), + ) + + module.validate_configuration_format(parser, config_format) + + +def test_validate_configuration_format_with_extra_option_should_raise(): + parser = flexmock() + parser.should_receive('sections').and_return(('section',)) + parser.should_receive('options').with_args('section').and_return(('option', 'extra')) + config_format = ( + module.Section_format( + 'section', + options=(module.Config_option('option', str, required=True),), + ), + ) + + with assert_raises(ValueError): + module.validate_configuration_format(parser, config_format) + + +def test_parse_section_options_should_return_section_options(): + parser = flexmock() + parser.should_receive('get').with_args('section', 'foo').and_return('value') + parser.should_receive('getint').with_args('section', 'bar').and_return(1) + parser.should_receive('has_option').with_args('section', 'foo').and_return(True) + parser.should_receive('has_option').with_args('section', 'bar').and_return(True) + + section_format = module.Section_format( + 'section', + ( + module.Config_option('foo', str, required=True), + module.Config_option('bar', int, required=True), + ), + ) + + config = module.parse_section_options(parser, section_format) + + assert config == OrderedDict( + ( + ('foo', 'value'), + ('bar', 1), + ) + ) + + +def insert_mock_parser(): parser = flexmock() parser.should_receive('readfp') - parser.should_receive('sections').and_return(section_names) flexmock(module).open = lambda filename: None flexmock(module).ConfigParser = parser return parser -def test_parse_configuration_should_return_config_data(): - section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION) - parser = insert_mock_parser(section_names) +def test_parse_configuration_should_return_section_configs(): + parser = insert_mock_parser() + mock_module = flexmock(module) + mock_module.should_receive('validate_configuration_format').with_args( + parser, module.CONFIG_FORMAT, + ).once() + mock_section_configs = (flexmock(), flexmock()) - for section_name in section_names: - parser.should_receive('options').with_args(section_name).and_return( - module.CONFIG_FORMAT[section_name], - ) + for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs): + mock_module.should_receive('parse_section_options').with_args( + parser, section_format, + ).and_return(section_config).once() - expected_config = ( - module.LocationConfig(flexmock(), flexmock()), - module.RetentionConfig(flexmock(), flexmock(), flexmock()), - ) - sections = ( - (module.CONFIG_SECTION_LOCATION, expected_config[0], 'get'), - (module.CONFIG_SECTION_RETENTION, expected_config[1], 'getint'), - ) + section_configs = module.parse_configuration('filename') - for section_name, section_config, method_name in sections: - for index, option_name in enumerate(module.CONFIG_FORMAT[section_name]): - ( - parser.should_receive(method_name).with_args(section_name, option_name) - .and_return(section_config[index]) - ) - - config = module.parse_configuration(flexmock()) - - assert config == expected_config - - -def test_parse_configuration_with_missing_section_should_raise(): - insert_mock_parser((module.CONFIG_SECTION_LOCATION,)) - - with assert_raises(ValueError): - module.parse_configuration(flexmock()) - - -def test_parse_configuration_with_extra_section_should_raise(): - insert_mock_parser((module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION, 'extra')) - - with assert_raises(ValueError): - module.parse_configuration(flexmock()) - - -def test_parse_configuration_with_missing_option_should_raise(): - section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION) - parser = insert_mock_parser(section_names) - - for section_name in section_names: - parser.should_receive('options').with_args(section_name).and_return( - module.CONFIG_FORMAT[section_name][:-1], - ) - - with assert_raises(ValueError): - module.parse_configuration(flexmock()) - - -def test_parse_configuration_with_extra_option_should_raise(): - section_names = (module.CONFIG_SECTION_LOCATION, module.CONFIG_SECTION_RETENTION) - parser = insert_mock_parser(section_names) - - for section_name in section_names: - parser.should_receive('options').with_args(section_name).and_return( - module.CONFIG_FORMAT[section_name] + ('extra',), - ) - - with assert_raises(ValueError): - module.parse_configuration(flexmock()) + assert section_configs == mock_section_configs diff --git a/sample/config b/sample/config index 3d7a111..aa2acf3 100644 --- a/sample/config +++ b/sample/config @@ -7,6 +7,11 @@ repository: user@backupserver:sourcehostname.attic [retention] # Retention policy for how many backups to keep in each category. +# See https://attic-backup.org/usage.html#attic-prune for details. +#keep_within: 3h +#keep_hourly: 24 keep_daily: 7 keep_weekly: 4 keep_monthly: 6 +keep_yearly: 1 +#prefix: sourcehostname diff --git a/setup.py b/setup.py index d23ba77..e737112 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='atticmatic', - version='0.0.1', + version='0.0.2', description='A wrapper script for Attic backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', From 18267b96771b2a64638e8dc1f8dbd1da3eca3bcd Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 6 Dec 2014 18:35:28 -0800 Subject: [PATCH 025/189] Added tag 0.0.2 for changeset 467d3a3ce918 --- .hgtags | 1 + 1 file changed, 1 insertion(+) create mode 100644 .hgtags diff --git a/.hgtags b/.hgtags new file mode 100644 index 0000000..ec58244 --- /dev/null +++ b/.hgtags @@ -0,0 +1 @@ +467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2 From 511314a54a50d371de7c358d1e6e8a337eb47875 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 20 Dec 2014 10:56:03 -0800 Subject: [PATCH 026/189] Adding a note about repository encryption. --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b02984b..034bbfe 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,12 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on To get up and running with Attic, follow the [Attic Quick Start](https://attic-backup.org/quickstart.html) guide to create an Attic -repository on a local or remote host. +repository on a local or remote host. Note that if you plan to run atticmatic +on a schedule with cron, and you encrypt your attic repository with a +passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE` +environment variable. See [attic's repository encryption +documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for +more info. If the repository is on a remote host, make sure that your local root user has key-based ssh access to the desired user account on the remote host. From dbd312981e4b2600905075ef8def4d1b56b59a02 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 20 Dec 2014 11:37:25 -0800 Subject: [PATCH 027/189] Integration tests for argument parsing. --- NEWS | 5 +++ atticmatic/command.py | 14 ++++--- atticmatic/tests/integration/__init__.py | 0 atticmatic/tests/integration/test_command.py | 40 ++++++++++++++++++++ 4 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 atticmatic/tests/integration/__init__.py create mode 100644 atticmatic/tests/integration/test_command.py diff --git a/NEWS b/NEWS index e9e6290..17bf8d3 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +default + + * Integration tests for argument parsing. + * Documentation updates about repository encryption. + 0.0.2 * Configuration support for additional attic prune flags: keep_within, keep_hourly, keep_yearly, diff --git a/atticmatic/command.py b/atticmatic/command.py index d47f63a..4623c47 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -7,7 +7,11 @@ from atticmatic.attic import create_archive, prune_archives from atticmatic.config import parse_configuration -def parse_arguments(): +DEFAULT_CONFIG_FILENAME = '/etc/atticmatic/config' +DEFAULT_EXCLUDES_FILENAME = '/etc/atticmatic/excludes' + + +def parse_arguments(*arguments): ''' Parse the command-line arguments from sys.argv and return them as an ArgumentParser instance. ''' @@ -15,13 +19,13 @@ def parse_arguments(): parser.add_argument( '-c', '--config', dest='config_filename', - default='/etc/atticmatic/config', + default=DEFAULT_CONFIG_FILENAME, help='Configuration filename', ) parser.add_argument( '--excludes', dest='excludes_filename', - default='/etc/atticmatic/excludes', + default=DEFAULT_EXCLUDES_FILENAME, help='Excludes filename', ) parser.add_argument( @@ -30,12 +34,12 @@ def parse_arguments(): help='Display verbose progress information', ) - return parser.parse_args() + return parser.parse_args(arguments) def main(): try: - args = parse_arguments() + args = parse_arguments(*sys.argv[1:]) location_config, retention_config = parse_configuration(args.config_filename) create_archive(args.excludes_filename, args.verbose, **location_config) diff --git a/atticmatic/tests/integration/__init__.py b/atticmatic/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/atticmatic/tests/integration/test_command.py b/atticmatic/tests/integration/test_command.py new file mode 100644 index 0000000..3a5e794 --- /dev/null +++ b/atticmatic/tests/integration/test_command.py @@ -0,0 +1,40 @@ +import sys + +from nose.tools import assert_raises + +from atticmatic import command as module + + +def test_parse_arguments_with_no_arguments_uses_defaults(): + parser = module.parse_arguments() + + assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME + assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME + assert parser.verbose == False + + +def test_parse_arguments_with_filename_arguments_overrides_defaults(): + parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes') + + assert parser.config_filename == 'myconfig' + assert parser.excludes_filename == 'myexcludes' + assert parser.verbose == False + + +def test_parse_arguments_with_verbose_flag_overrides_default(): + parser = module.parse_arguments('--verbose') + + assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME + assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME + assert parser.verbose == True + + +def test_parse_arguments_with_invalid_arguments_exits(): + original_stderr = sys.stderr + sys.stderr = sys.stdout + + try: + with assert_raises(SystemExit): + module.parse_arguments('--posix-me-harder') + finally: + sys.stderr = original_stderr From b1113d57ae96db68805edc045c9c21e63a253d8d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 20 Dec 2014 11:42:27 -0800 Subject: [PATCH 028/189] Correcting doc string based on updated command-line arguments source. --- atticmatic/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atticmatic/command.py b/atticmatic/command.py index 4623c47..ff31ca7 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -13,7 +13,7 @@ DEFAULT_EXCLUDES_FILENAME = '/etc/atticmatic/excludes' def parse_arguments(*arguments): ''' - Parse the command-line arguments from sys.argv and return them as an ArgumentParser instance. + Parse the given command-line arguments and return them as an ArgumentParser instance. ''' parser = ArgumentParser() parser.add_argument( From eaf2bd22c1eabf00749d3ef6c875e4b47adc1a28 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 14 Feb 2015 09:23:40 -0800 Subject: [PATCH 029/189] After pruning, run attic's consistency checks on all archives. --- NEWS | 3 ++- README.md | 13 ++++++----- atticmatic/attic.py | 18 ++++++++++++++- atticmatic/command.py | 6 +++-- atticmatic/tests/unit/test_attic.py | 35 +++++++++++++++++++++++++++-- 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/NEWS b/NEWS index 17bf8d3..93d33a5 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ -default +0.0.3 + * After pruning, run attic's consistency checks on all archives. * Integration tests for argument parsing. * Documentation updates about repository encryption. diff --git a/README.md b/README.md index 034bbfe..4a2d211 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,11 @@ save_as: atticmatic/index.html ## Overview atticmatic is a simple Python wrapper script for the [Attic backup -software](https://attic-backup.org/) that initiates a backup and prunes any -old backups according to a retention policy. The script supports specifying -your settings in a declarative configuration file rather than having to put -them all on the command-line, and handles common errors. +software](https://attic-backup.org/) that initiates a backup, prunes any old +backups according to a retention policy, and validates backups for +consistency. The script supports specifying your settings in a declarative +configuration file rather than having to put them all on the command-line, and +handles common errors. Here's an example config file: @@ -68,7 +69,9 @@ arguments: atticmatic -This will also prune any old backups as per the configured retention policy. +This will also prune any old backups as per the configured retention policy, +and check backups for consistency problems due to things like file damage. + By default, the backup will proceed silently except in the case of errors. But if you'd like to to get additional information about the progress of the backup as it proceeds, use the verbose option instead: diff --git a/atticmatic/attic.py b/atticmatic/attic.py index c884210..d0c678d 100644 --- a/atticmatic/attic.py +++ b/atticmatic/attic.py @@ -1,5 +1,5 @@ from datetime import datetime - +import os import platform import subprocess @@ -63,3 +63,19 @@ def prune_archives(verbose, repository, retention_config): ) + (('--verbose',) if verbose else ()) subprocess.check_call(command) + + +def check_archives(verbose, repository): + ''' + Given a verbosity flag and a local or remote repository path, check the contained attic archives + for consistency. + ''' + command = ( + 'attic', 'check', + repository, + ) + (('--verbose',) if verbose else ()) + + # Attic's check command spews to stdout even without the verbose flag. Suppress it. + stdout = None if verbose else open(os.devnull, 'w') + + subprocess.check_call(command, stdout=stdout) diff --git a/atticmatic/command.py b/atticmatic/command.py index ff31ca7..92976f0 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -3,7 +3,7 @@ from argparse import ArgumentParser from subprocess import CalledProcessError import sys -from atticmatic.attic import create_archive, prune_archives +from atticmatic.attic import check_archives, create_archive, prune_archives from atticmatic.config import parse_configuration @@ -41,9 +41,11 @@ def main(): try: args = parse_arguments(*sys.argv[1:]) location_config, retention_config = parse_configuration(args.config_filename) + repository = location_config['repository'] create_archive(args.excludes_filename, args.verbose, **location_config) - prune_archives(args.verbose, location_config['repository'], retention_config) + prune_archives(args.verbose, repository, retention_config) + check_archives(args.verbose, repository) except (ValueError, IOError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py index 44b38bb..2c93e8c 100644 --- a/atticmatic/tests/unit/test_attic.py +++ b/atticmatic/tests/unit/test_attic.py @@ -5,9 +5,9 @@ from flexmock import flexmock from atticmatic import attic as module -def insert_subprocess_mock(check_call_command): +def insert_subprocess_mock(check_call_command, **kwargs): subprocess = flexmock() - subprocess.should_receive('check_call').with_args(check_call_command).once() + subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() flexmock(module).subprocess = subprocess @@ -111,3 +111,34 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters() verbose=True, retention_config=retention_config, ) + + +def test_check_archives_should_call_attic_with_parameters(): + stdout = flexmock() + insert_subprocess_mock( + ('attic', 'check', 'repo'), + stdout=stdout, + ) + insert_platform_mock() + insert_datetime_mock() + flexmock(module).open = lambda filename, mode: stdout + flexmock(module).os = flexmock().should_receive('devnull').mock + + module.check_archives( + verbose=False, + repository='repo', + ) + + +def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters(): + insert_subprocess_mock( + ('attic', 'check', 'repo', '--verbose'), + stdout=None, + ) + insert_platform_mock() + insert_datetime_mock() + + module.check_archives( + verbose=True, + repository='repo', + ) From 9f5dd6c10deebff35c7eddaed1295a7ec5a71a0d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 14 Feb 2015 09:24:15 -0800 Subject: [PATCH 030/189] Added tag 0.0.3 for changeset 7730ae34665c --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index ec58244..3f99142 100644 --- a/.hgtags +++ b/.hgtags @@ -1 +1,2 @@ 467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2 +7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3 From f23810f19aa2d883453d390476d762892c631663 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 14 Feb 2015 09:31:42 -0800 Subject: [PATCH 031/189] Updating install instructions so you can upgrade from one release of atticmatic to the next. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a2d211..99b8a56 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ key-based ssh access to the desired user account on the remote host. To install atticmatic, run the following command to download and install it: - sudo pip install hg+https://torsion.org/hg/atticmatic + sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic Then copy the following configuration files: From 02df59e964675ccabe9a30f375a2f6dd886ee9b0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 28 Feb 2015 11:03:22 -0800 Subject: [PATCH 032/189] Added a troubleshooting section with steps to deal with broken pipes. --- NEWS | 4 ++++ README.md | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/NEWS b/NEWS index 93d33a5..0d2cb0e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.0.4-dev + + * Added a troubleshooting section with steps to deal with broken pipes. + 0.0.3 * After pruning, run attic's consistency checks on all archives. diff --git a/README.md b/README.md index 99b8a56..d289369 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,27 @@ To actually run tests, run: nosetests --detailed-errors +## Troubleshooting + +### Broken pipe with remote repository + +When running atticmatic on a large remote repository, you may receive errors +like the following, particularly while "attic check" is valiating backups for +consistency: + + Write failed: Broken pipe + attic: Error: Connection closed by remote host + +This error can be caused by an ssh timeout, which you can rectify by adding +the following to the ~/.ssh/config file on the client: + + Host * + ServerAliveInterval 120 + +This should make the client keep the connection alive while validating +backups. + + ## Feedback Questions? Comments? Got a patch? Contact . From 2639b7105a28307a0a34c1728e1cecff1755c245 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 09:41:58 -0700 Subject: [PATCH 033/189] Added nosetests config file (setup.cfg) with defaults. --- NEWS | 1 + README.md | 2 +- setup.cfg | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 setup.cfg diff --git a/NEWS b/NEWS index 0d2cb0e..c2a9dff 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 0.0.4-dev * Added a troubleshooting section with steps to deal with broken pipes. + * Added nosetests config file (setup.cfg) with defaults. 0.0.3 diff --git a/README.md b/README.md index d289369..825ca5e 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ To install test-specific dependencies, first run: To actually run tests, run: - nosetests --detailed-errors + nosetests ## Troubleshooting diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..ead29d1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[nosetests] +detailed-errors=1 From aa48b95ee7dadac145e9bf513b38e440f9a7bb68 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 09:52:40 -0700 Subject: [PATCH 034/189] Bumping setup.py version. --- NEWS | 1 + README.md | 31 ++++++----- atticmatic/attic.py | 12 ++++- atticmatic/command.py | 2 +- atticmatic/tests/unit/test_attic.py | 81 +++++++++++++++++++++++++---- setup.py | 2 +- 6 files changed, 104 insertions(+), 25 deletions(-) diff --git a/NEWS b/NEWS index c2a9dff..1464d2b 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 0.0.4-dev + * Helpful error message about how to create a repository if one is missing. * Added a troubleshooting section with steps to deal with broken pipes. * Added nosetests config file (setup.cfg) with defaults. diff --git a/README.md b/README.md index 825ca5e..cd4298d 100644 --- a/README.md +++ b/README.md @@ -37,18 +37,6 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on ## Setup -To get up and running with Attic, follow the [Attic Quick -Start](https://attic-backup.org/quickstart.html) guide to create an Attic -repository on a local or remote host. Note that if you plan to run atticmatic -on a schedule with cron, and you encrypt your attic repository with a -passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE` -environment variable. See [attic's repository encryption -documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for -more info. - -If the repository is on a remote host, make sure that your local root user has -key-based ssh access to the desired user account on the remote host. - To install atticmatic, run the following command to download and install it: sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic @@ -59,7 +47,24 @@ Then copy the following configuration files: sudo mkdir /etc/atticmatic/ sudo cp sample/config sample/excludes /etc/atticmatic/ -Lastly, modify those files with your desired configuration. +Modify those files with your desired configuration, including the path to an +attic repository. + +If you don't yet have an attic repository, then the first time you run +atticmatic, you'll get an error with information on how to create a repository +on a local or remote host. + +And if the repository is on a remote host, make sure that your local root user +has key-based ssh access to the desired user account on the remote host. + +It is recommended that you create your attic repository with keyfile +encryption, as passphrase-based encryption is less suited for automated +backups. If you do plan to run atticmatic on a schedule with cron, and you +encrypt your attic repository with a passphrase instead of a key file, you'll +need to set the `ATTIC_PASSPHRASE` environment variable. See [attic's +repository encryption +documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for +more info. ## Usage diff --git a/atticmatic/attic.py b/atticmatic/attic.py index d0c678d..f0c9e7e 100644 --- a/atticmatic/attic.py +++ b/atticmatic/attic.py @@ -1,7 +1,10 @@ +from __future__ import print_function from datetime import datetime import os import platform +import re import subprocess +import sys def create_archive(excludes_filename, verbose, source_directories, repository): @@ -23,7 +26,14 @@ def create_archive(excludes_filename, verbose, source_directories, repository): ('--verbose', '--stats') if verbose else () ) - subprocess.check_call(command) + try: + subprocess.check_output(command, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError, error: + print(error.output.strip(), file=sys.stderr) + + if re.search('Error: Repository .* does not exist', error.output): + raise RuntimeError('To create a repository, run: attic init --encryption=keyfile {}'.format(repository)) + raise error def make_prune_flags(retention_config): diff --git a/atticmatic/command.py b/atticmatic/command.py index 92976f0..e0f4eb1 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -46,6 +46,6 @@ def main(): create_archive(args.excludes_filename, args.verbose, **location_config) prune_archives(args.verbose, repository, retention_config) check_archives(args.verbose, repository) - except (ValueError, IOError, CalledProcessError) as error: + except (ValueError, IOError, CalledProcessError, RuntimeError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py index 2c93e8c..30431cc 100644 --- a/atticmatic/tests/unit/test_attic.py +++ b/atticmatic/tests/unit/test_attic.py @@ -1,14 +1,44 @@ from collections import OrderedDict +import sys from flexmock import flexmock +from nose.tools import assert_raises from atticmatic import attic as module -def insert_subprocess_mock(check_call_command, **kwargs): - subprocess = flexmock() - subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() +class MockCalledProcessError(Exception): + def __init__(self, output): + self.output = output + + +def insert_subprocess_check_output_mock(call_command, error_output=None, **kwargs): + subprocess = flexmock(CalledProcessError=MockCalledProcessError, STDOUT=flexmock()) + + expectation = subprocess.should_receive('check_output').with_args( + call_command, + stderr=subprocess.STDOUT, + **kwargs + ).once() + + if error_output: + expectation.and_raise(MockCalledProcessError, output=error_output) + flexmock(sys.modules['__builtin__']).should_receive('print') + flexmock(module).subprocess = subprocess + return subprocess + + +def insert_subprocess_check_call_mock(call_command, **kwargs): + subprocess = flexmock() + + subprocess.should_receive('check_call').with_args( + call_command, + **kwargs + ).once() + + flexmock(module).subprocess = subprocess + return subprocess def insert_platform_mock(): @@ -22,7 +52,7 @@ def insert_datetime_mock(): def test_create_archive_should_call_attic_with_parameters(): - insert_subprocess_mock( + insert_subprocess_check_output_mock( ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'), ) insert_platform_mock() @@ -37,7 +67,7 @@ def test_create_archive_should_call_attic_with_parameters(): def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters(): - insert_subprocess_mock( + insert_subprocess_check_output_mock( ( 'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar', '--verbose', '--stats', @@ -53,6 +83,39 @@ def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters() repository='repo', ) +def test_create_archive_with_missing_repository_should_raise(): + insert_subprocess_check_output_mock( + ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'), + error_output='Error: Repository repo does not exist', + ) + insert_platform_mock() + insert_datetime_mock() + + with assert_raises(RuntimeError): + module.create_archive( + excludes_filename='excludes', + verbose=False, + source_directories='foo bar', + repository='repo', + ) + + +def test_create_archive_with_other_error_should_raise(): + subprocess = insert_subprocess_check_output_mock( + ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'), + error_output='Something went wrong', + ) + insert_platform_mock() + insert_datetime_mock() + + with assert_raises(subprocess.CalledProcessError): + module.create_archive( + excludes_filename='excludes', + verbose=False, + source_directories='foo bar', + repository='repo', + ) + BASE_PRUNE_FLAGS = ( ('--keep-daily', '1'), @@ -80,7 +143,7 @@ def test_prune_archives_should_call_attic_with_parameters(): flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) - insert_subprocess_mock( + insert_subprocess_check_call_mock( ( 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', @@ -99,7 +162,7 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters() flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) - insert_subprocess_mock( + insert_subprocess_check_call_mock( ( 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', '--verbose', @@ -115,7 +178,7 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters() def test_check_archives_should_call_attic_with_parameters(): stdout = flexmock() - insert_subprocess_mock( + insert_subprocess_check_call_mock( ('attic', 'check', 'repo'), stdout=stdout, ) @@ -131,7 +194,7 @@ def test_check_archives_should_call_attic_with_parameters(): def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters(): - insert_subprocess_mock( + insert_subprocess_check_call_mock( ('attic', 'check', 'repo', '--verbose'), stdout=None, ) diff --git a/setup.py b/setup.py index e737112..9bcd56d 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='atticmatic', - version='0.0.2', + version='0.0.4', description='A wrapper script for Attic backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', From ee5697ac374619f8d640215cb07fa789b0f466f3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 10:14:16 -0700 Subject: [PATCH 035/189] Fixing Python 3 test incompatibility with builtins. --- atticmatic/attic.py | 2 +- atticmatic/tests/unit/test_attic.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/atticmatic/attic.py b/atticmatic/attic.py index f0c9e7e..d0b1cc8 100644 --- a/atticmatic/attic.py +++ b/atticmatic/attic.py @@ -28,7 +28,7 @@ def create_archive(excludes_filename, verbose, source_directories, repository): try: subprocess.check_output(command, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError, error: + except subprocess.CalledProcessError as error: print(error.output.strip(), file=sys.stderr) if re.search('Error: Repository .* does not exist', error.output): diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py index 30431cc..e54f445 100644 --- a/atticmatic/tests/unit/test_attic.py +++ b/atticmatic/tests/unit/test_attic.py @@ -1,5 +1,10 @@ from collections import OrderedDict -import sys +try: + # Python 2 + import __builtin__ as builtins +except ImportError: + # Python 3 + import builtins from flexmock import flexmock from nose.tools import assert_raises @@ -23,7 +28,7 @@ def insert_subprocess_check_output_mock(call_command, error_output=None, **kwarg if error_output: expectation.and_raise(MockCalledProcessError, output=error_output) - flexmock(sys.modules['__builtin__']).should_receive('print') + flexmock(builtins).should_receive('print') flexmock(module).subprocess = subprocess return subprocess From 715b240589c76ff03cdedc9e74899a95e92b1352 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 10:14:30 -0700 Subject: [PATCH 036/189] Now using tox to run tests against multiple versions of Python in one go. --- .hgignore | 1 + NEWS | 5 +++-- README.md | 8 ++++---- test_requirements.txt | 2 ++ tox.ini | 8 ++++++++ 5 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 test_requirements.txt create mode 100644 tox.ini diff --git a/.hgignore b/.hgignore index abab077..d7c266a 100644 --- a/.hgignore +++ b/.hgignore @@ -2,3 +2,4 @@ syntax: glob *.egg-info *.pyc *.swp +.tox diff --git a/NEWS b/NEWS index 1464d2b..624ad27 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ 0.0.4-dev + * Now using tox to run tests against multiple versions of Python in one go. * Helpful error message about how to create a repository if one is missing. - * Added a troubleshooting section with steps to deal with broken pipes. - * Added nosetests config file (setup.cfg) with defaults. + * Troubleshooting section with steps to deal with broken pipes. + * Nosetests config file (setup.cfg) with defaults. 0.0.3 diff --git a/README.md b/README.md index cd4298d..c608273 100644 --- a/README.md +++ b/README.md @@ -90,13 +90,13 @@ If you'd like to see the available command-line arguments, view the help: ## Running tests -To install test-specific dependencies, first run: +First install tox, which is used for setting up testing environments: - sudo python setup.py test + pip install tox -To actually run tests, run: +Then, to actually run tests, run: - nosetests + tox ## Troubleshooting diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 0000000..5a34a85 --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,2 @@ +flexmock==0.9.7 +nose==1.3.4 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..daf5e8d --- /dev/null +++ b/tox.ini @@ -0,0 +1,8 @@ +[tox] +envlist=py27,py34 +skipsdist=True + +[testenv] +usedevelop=True +deps=-rtest_requirements.txt +commands = nosetests From 66286f92df9275e8e614d7aac1f8b192041d5cfb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 10:15:03 -0700 Subject: [PATCH 037/189] Releasing 0.0.4. --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 624ad27..e5ba534 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -0.0.4-dev +0.0.4 * Now using tox to run tests against multiple versions of Python in one go. * Helpful error message about how to create a repository if one is missing. From d9e396e26415a1df5047ea40dcf9e9e6a383b532 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 10:19:12 -0700 Subject: [PATCH 038/189] Added tag 0.0.4 for changeset 4bb2e81fc770 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 3f99142..d101a2a 100644 --- a/.hgtags +++ b/.hgtags @@ -1,2 +1,3 @@ 467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2 7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3 +4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4 From bda6451c1d6be0ff105f83d32aca4b4ba516a0cb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 10:39:08 -0700 Subject: [PATCH 039/189] Added tag 0.0.5 for changeset b31d51b63370 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index d101a2a..a4c5726 100644 --- a/.hgtags +++ b/.hgtags @@ -1,3 +1,4 @@ 467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2 7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3 4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4 +b31d51b633701554e84f996cc0c73bad2990780b 0.0.5 From ac6c927a23865411da85f489e60afffc4c0db7da Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 10:44:18 -0700 Subject: [PATCH 040/189] Backout out "helpful" error message that broke --verbose. --- NEWS | 5 ++ README.md | 31 +++++------ atticmatic/attic.py | 12 +---- atticmatic/command.py | 2 +- atticmatic/tests/unit/test_attic.py | 84 +++-------------------------- setup.py | 2 +- 6 files changed, 29 insertions(+), 107 deletions(-) diff --git a/NEWS b/NEWS index e5ba534..43351a5 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +0.0.5 + + * Fixed regression with --verbose output being buffered. This means dropping the helpful error + message introduced in 0.0.4. + 0.0.4 * Now using tox to run tests against multiple versions of Python in one go. diff --git a/README.md b/README.md index c608273..f399646 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,18 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on ## Setup +To get up and running with Attic, follow the [Attic Quick +Start](https://attic-backup.org/quickstart.html) guide to create an Attic +repository on a local or remote host. Note that if you plan to run atticmatic +on a schedule with cron, and you encrypt your attic repository with a +passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE` +environment variable. See [attic's repository encryption +documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for +more info. + +If the repository is on a remote host, make sure that your local root user has +key-based ssh access to the desired user account on the remote host. + To install atticmatic, run the following command to download and install it: sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic @@ -47,24 +59,7 @@ Then copy the following configuration files: sudo mkdir /etc/atticmatic/ sudo cp sample/config sample/excludes /etc/atticmatic/ -Modify those files with your desired configuration, including the path to an -attic repository. - -If you don't yet have an attic repository, then the first time you run -atticmatic, you'll get an error with information on how to create a repository -on a local or remote host. - -And if the repository is on a remote host, make sure that your local root user -has key-based ssh access to the desired user account on the remote host. - -It is recommended that you create your attic repository with keyfile -encryption, as passphrase-based encryption is less suited for automated -backups. If you do plan to run atticmatic on a schedule with cron, and you -encrypt your attic repository with a passphrase instead of a key file, you'll -need to set the `ATTIC_PASSPHRASE` environment variable. See [attic's -repository encryption -documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for -more info. +Lastly, modify those files with your desired configuration. ## Usage diff --git a/atticmatic/attic.py b/atticmatic/attic.py index d0b1cc8..d0c678d 100644 --- a/atticmatic/attic.py +++ b/atticmatic/attic.py @@ -1,10 +1,7 @@ -from __future__ import print_function from datetime import datetime import os import platform -import re import subprocess -import sys def create_archive(excludes_filename, verbose, source_directories, repository): @@ -26,14 +23,7 @@ def create_archive(excludes_filename, verbose, source_directories, repository): ('--verbose', '--stats') if verbose else () ) - try: - subprocess.check_output(command, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as error: - print(error.output.strip(), file=sys.stderr) - - if re.search('Error: Repository .* does not exist', error.output): - raise RuntimeError('To create a repository, run: attic init --encryption=keyfile {}'.format(repository)) - raise error + subprocess.check_call(command) def make_prune_flags(retention_config): diff --git a/atticmatic/command.py b/atticmatic/command.py index e0f4eb1..92976f0 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -46,6 +46,6 @@ def main(): create_archive(args.excludes_filename, args.verbose, **location_config) prune_archives(args.verbose, repository, retention_config) check_archives(args.verbose, repository) - except (ValueError, IOError, CalledProcessError, RuntimeError) as error: + except (ValueError, IOError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py index e54f445..2c93e8c 100644 --- a/atticmatic/tests/unit/test_attic.py +++ b/atticmatic/tests/unit/test_attic.py @@ -1,49 +1,14 @@ from collections import OrderedDict -try: - # Python 2 - import __builtin__ as builtins -except ImportError: - # Python 3 - import builtins from flexmock import flexmock -from nose.tools import assert_raises from atticmatic import attic as module -class MockCalledProcessError(Exception): - def __init__(self, output): - self.output = output - - -def insert_subprocess_check_output_mock(call_command, error_output=None, **kwargs): - subprocess = flexmock(CalledProcessError=MockCalledProcessError, STDOUT=flexmock()) - - expectation = subprocess.should_receive('check_output').with_args( - call_command, - stderr=subprocess.STDOUT, - **kwargs - ).once() - - if error_output: - expectation.and_raise(MockCalledProcessError, output=error_output) - flexmock(builtins).should_receive('print') - - flexmock(module).subprocess = subprocess - return subprocess - - -def insert_subprocess_check_call_mock(call_command, **kwargs): +def insert_subprocess_mock(check_call_command, **kwargs): subprocess = flexmock() - - subprocess.should_receive('check_call').with_args( - call_command, - **kwargs - ).once() - + subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() flexmock(module).subprocess = subprocess - return subprocess def insert_platform_mock(): @@ -57,7 +22,7 @@ def insert_datetime_mock(): def test_create_archive_should_call_attic_with_parameters(): - insert_subprocess_check_output_mock( + insert_subprocess_mock( ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'), ) insert_platform_mock() @@ -72,7 +37,7 @@ def test_create_archive_should_call_attic_with_parameters(): def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters(): - insert_subprocess_check_output_mock( + insert_subprocess_mock( ( 'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar', '--verbose', '--stats', @@ -88,39 +53,6 @@ def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters() repository='repo', ) -def test_create_archive_with_missing_repository_should_raise(): - insert_subprocess_check_output_mock( - ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'), - error_output='Error: Repository repo does not exist', - ) - insert_platform_mock() - insert_datetime_mock() - - with assert_raises(RuntimeError): - module.create_archive( - excludes_filename='excludes', - verbose=False, - source_directories='foo bar', - repository='repo', - ) - - -def test_create_archive_with_other_error_should_raise(): - subprocess = insert_subprocess_check_output_mock( - ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'), - error_output='Something went wrong', - ) - insert_platform_mock() - insert_datetime_mock() - - with assert_raises(subprocess.CalledProcessError): - module.create_archive( - excludes_filename='excludes', - verbose=False, - source_directories='foo bar', - repository='repo', - ) - BASE_PRUNE_FLAGS = ( ('--keep-daily', '1'), @@ -148,7 +80,7 @@ def test_prune_archives_should_call_attic_with_parameters(): flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) - insert_subprocess_check_call_mock( + insert_subprocess_mock( ( 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', @@ -167,7 +99,7 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters() flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) - insert_subprocess_check_call_mock( + insert_subprocess_mock( ( 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', '--verbose', @@ -183,7 +115,7 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters() def test_check_archives_should_call_attic_with_parameters(): stdout = flexmock() - insert_subprocess_check_call_mock( + insert_subprocess_mock( ('attic', 'check', 'repo'), stdout=stdout, ) @@ -199,7 +131,7 @@ def test_check_archives_should_call_attic_with_parameters(): def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters(): - insert_subprocess_check_call_mock( + insert_subprocess_mock( ('attic', 'check', 'repo', '--verbose'), stdout=None, ) diff --git a/setup.py b/setup.py index 9bcd56d..e737112 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='atticmatic', - version='0.0.4', + version='0.0.2', description='A wrapper script for Attic backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', From 3506819511f01612f781d1a5baa7c2fcdf931b98 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 10:44:25 -0700 Subject: [PATCH 041/189] Added tag 0.0.5 for changeset aa8a807f4ba2 --- .hgtags | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.hgtags b/.hgtags index a4c5726..023f01f 100644 --- a/.hgtags +++ b/.hgtags @@ -2,3 +2,5 @@ 7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3 4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4 b31d51b633701554e84f996cc0c73bad2990780b 0.0.5 +b31d51b633701554e84f996cc0c73bad2990780b 0.0.5 +aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5 From aa1178dc492382379ee20a772d3150e154506358 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 10:46:55 -0700 Subject: [PATCH 042/189] Added tag 0.0.5 for changeset 569aef47a9b2 --- .hgtags | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.hgtags b/.hgtags index 023f01f..1d304c7 100644 --- a/.hgtags +++ b/.hgtags @@ -4,3 +4,5 @@ b31d51b633701554e84f996cc0c73bad2990780b 0.0.5 b31d51b633701554e84f996cc0c73bad2990780b 0.0.5 aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5 +aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5 +569aef47a9b25c55b13753f94706f5d330219995 0.0.5 From cb402d68464d78da6d47ba3bfa09f482d4482852 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 10:47:49 -0700 Subject: [PATCH 043/189] Re-fixing version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e737112..a771454 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='atticmatic', - version='0.0.2', + version='0.0.5', description='A wrapper script for Attic backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', From 4e4f8c2670dc4a25d0b8b2d84c18b9cdc54e9855 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 10:47:58 -0700 Subject: [PATCH 044/189] Added tag 0.0.5 for changeset a03495a8e8b4 --- .hgtags | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.hgtags b/.hgtags index 1d304c7..f3659ea 100644 --- a/.hgtags +++ b/.hgtags @@ -6,3 +6,5 @@ b31d51b633701554e84f996cc0c73bad2990780b 0.0.5 aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5 aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5 569aef47a9b25c55b13753f94706f5d330219995 0.0.5 +569aef47a9b25c55b13753f94706f5d330219995 0.0.5 +a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5 From 7750d2568c8d118d446181482518a9aba9317c2b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Mar 2015 11:15:40 -0700 Subject: [PATCH 045/189] Passing through command-line options from tox to nosetests. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index daf5e8d..6ac577b 100644 --- a/tox.ini +++ b/tox.ini @@ -5,4 +5,4 @@ skipsdist=True [testenv] usedevelop=True deps=-rtest_requirements.txt -commands = nosetests +commands = nosetests [] From cfd61dc1d15cc41d71c775197d17c07d52394978 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 10 May 2015 22:00:31 -0700 Subject: [PATCH 046/189] New configuration section for customizing which Attic consistency checks run, if any. --- NEWS | 4 ++ README.md | 3 ++ atticmatic/attic.py | 67 +++++++++++++++++++++++--- atticmatic/command.py | 10 ++-- atticmatic/config.py | 51 +++++++++++++++----- atticmatic/tests/unit/test_attic.py | 70 ++++++++++++++++++++++++++-- atticmatic/tests/unit/test_config.py | 70 +++++++++++++++++++++++++--- sample/config | 10 +++- setup.py | 2 +- 9 files changed, 251 insertions(+), 36 deletions(-) diff --git a/NEWS b/NEWS index 43351a5..5dbc3bc 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.0.6 + + * New configuration section for customizing which Attic consistency checks run, if any. + 0.0.5 * Fixed regression with --verbose output being buffered. This means dropping the helpful error diff --git a/README.md b/README.md index f399646..ee9cb72 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,9 @@ Here's an example config file: keep_weekly: 4 keep_monthly: 6 + [consistency] + checks: repository archives + Additionally, exclude patterns can be specified in a separate excludes config file, one pattern per line. diff --git a/atticmatic/attic.py b/atticmatic/attic.py index d0c678d..fc83997 100644 --- a/atticmatic/attic.py +++ b/atticmatic/attic.py @@ -26,7 +26,7 @@ def create_archive(excludes_filename, verbose, source_directories, repository): subprocess.check_call(command) -def make_prune_flags(retention_config): +def _make_prune_flags(retention_config): ''' Given a retention config dict mapping from option name to value, tranform it into an iterable of command-line name-value flag pairs. @@ -58,22 +58,77 @@ def prune_archives(verbose, repository, retention_config): repository, ) + tuple( element - for pair in make_prune_flags(retention_config) + for pair in _make_prune_flags(retention_config) for element in pair ) + (('--verbose',) if verbose else ()) subprocess.check_call(command) -def check_archives(verbose, repository): +DEFAULT_CHECKS = ('repository', 'archives') + + +def _parse_checks(consistency_config): ''' - Given a verbosity flag and a local or remote repository path, check the contained attic archives - for consistency. + Given a consistency config with a space-separated "checks" option, transform it to a tuple of + named checks to run. + + For example, given a retention config of: + + {'checks': 'repository archives'} + + This will be returned as: + + ('repository', 'archives') + + If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string + "disabled", return an empty tuple, meaning that no checks should be run. ''' + checks = consistency_config.get('checks', '').strip() + if not checks: + return DEFAULT_CHECKS + + return tuple( + check for check in consistency_config['checks'].split(' ') + if check.lower() not in ('disabled', '') + ) + + +def _make_check_flags(checks): + ''' + Given a parsed sequence of checks, transform it into tuple of command-line flags. + + For example, given parsed checks of: + + ('repository',) + + This will be returned as: + + ('--repository-only',) + ''' + if checks == DEFAULT_CHECKS: + return () + + return tuple( + '--{}-only'.format(check) for check in checks + ) + + +def check_archives(verbose, repository, consistency_config): + ''' + Given a verbosity flag, a local or remote repository path, and a consistency config dict, check + the contained attic archives for consistency. + + If there are no consistency checks to run, skip running them. + ''' + checks = _parse_checks(consistency_config) + if not checks: + return + command = ( 'attic', 'check', repository, - ) + (('--verbose',) if verbose else ()) + ) + _make_check_flags(checks) + (('--verbose',) if verbose else ()) # Attic's check command spews to stdout even without the verbose flag. Suppress it. stdout = None if verbose else open(os.devnull, 'w') diff --git a/atticmatic/command.py b/atticmatic/command.py index 92976f0..0a844b2 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -40,12 +40,12 @@ def parse_arguments(*arguments): def main(): try: args = parse_arguments(*sys.argv[1:]) - location_config, retention_config = parse_configuration(args.config_filename) - repository = location_config['repository'] + config = parse_configuration(args.config_filename) + repository = config.location['repository'] - create_archive(args.excludes_filename, args.verbose, **location_config) - prune_archives(args.verbose, repository, retention_config) - check_archives(args.verbose, repository) + create_archive(args.excludes_filename, args.verbose, **config.location) + prune_archives(args.verbose, repository, config.retention) + check_archives(args.verbose, repository, config.consistency) except (ValueError, IOError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/atticmatic/config.py b/atticmatic/config.py index f953860..8254c88 100644 --- a/atticmatic/config.py +++ b/atticmatic/config.py @@ -39,6 +39,12 @@ CONFIG_FORMAT = ( option('keep_yearly', int, required=False), option('prefix', required=False), ), + ), + Section_format( + 'consistency', + ( + option('checks', required=False), + ), ) ) @@ -49,20 +55,34 @@ def validate_configuration_format(parser, config_format): configuration file has the expected sections, that any required options are present in those sections, and that there aren't any unexpected options. + A section is required if any of its contained options are required. + Raise ValueError if anything is awry. ''' - section_names = parser.sections() - required_section_names = tuple(section.name for section in config_format) + section_names = set(parser.sections()) + required_section_names = tuple( + section.name for section in config_format + if any(option.required for option in section.options) + ) - if set(section_names) != set(required_section_names): + unknown_section_names = section_names - set( + section_format.name for section_format in config_format + ) + if unknown_section_names: raise ValueError( - 'Expected config sections {} but found sections: {}'.format( - ', '.join(required_section_names), - ', '.join(section_names) - ) + 'Unknown config sections found: {}'.format(', '.join(unknown_section_names)) + ) + + missing_section_names = set(required_section_names) - section_names + if missing_section_names: + raise ValueError( + 'Missing config sections: {}'.format(', '.join(missing_section_names)) ) for section_format in config_format: + if section_format.name not in section_names: + continue + option_names = parser.options(section_format.name) expected_options = section_format.options @@ -90,6 +110,11 @@ def validate_configuration_format(parser, config_format): ) +# Describes a parsed configuration, where each attribute is the name of a configuration file section +# and each value is a dict of that section's parsed options. +Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT)) + + def parse_section_options(parser, section_format): ''' Given an open ConfigParser and an expected section format, return the option values from that @@ -112,8 +137,8 @@ def parse_section_options(parser, section_format): def parse_configuration(config_filename): ''' - Given a config filename of the expected format, return the parsed configuration as a tuple of - (location config, retention config) where each config is a dict of that section's options. + Given a config filename of the expected format, return the parsed configuration as Parsed_config + data structure. Raise IOError if the file cannot be read, or ValueError if the format is not as expected. ''' @@ -122,7 +147,9 @@ def parse_configuration(config_filename): validate_configuration_format(parser, CONFIG_FORMAT) - return tuple( - parse_section_options(parser, section_format) - for section_format in CONFIG_FORMAT + return Parsed_config( + *( + parse_section_options(parser, section_format) + for section_format in CONFIG_FORMAT + ) ) diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py index 2c93e8c..26c6299 100644 --- a/atticmatic/tests/unit/test_attic.py +++ b/atticmatic/tests/unit/test_attic.py @@ -11,6 +11,12 @@ def insert_subprocess_mock(check_call_command, **kwargs): flexmock(module).subprocess = subprocess +def insert_subprocess_never(): + subprocess = flexmock() + subprocess.should_receive('check_call').never() + flexmock(module).subprocess = subprocess + + def insert_platform_mock(): flexmock(module).platform = flexmock().should_receive('node').and_return('host').mock @@ -70,14 +76,14 @@ def test_make_prune_flags_should_return_flags_from_config(): ) ) - result = module.make_prune_flags(retention_config) + result = module._make_prune_flags(retention_config) assert tuple(result) == BASE_PRUNE_FLAGS def test_prune_archives_should_call_attic_with_parameters(): retention_config = flexmock() - flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return( + flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) insert_subprocess_mock( @@ -96,7 +102,7 @@ def test_prune_archives_should_call_attic_with_parameters(): def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters(): retention_config = flexmock() - flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return( + flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) insert_subprocess_mock( @@ -113,7 +119,46 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters() ) +def test_parse_checks_returns_them_as_tuple(): + checks = module._parse_checks({'checks': 'foo disabled bar'}) + + assert checks == ('foo', 'bar') + + +def test_parse_checks_with_missing_value_returns_defaults(): + checks = module._parse_checks({}) + + assert checks == module.DEFAULT_CHECKS + + +def test_parse_checks_with_blank_value_returns_defaults(): + checks = module._parse_checks({'checks': ''}) + + assert checks == module.DEFAULT_CHECKS + + +def test_parse_checks_with_disabled_returns_no_checks(): + checks = module._parse_checks({'checks': 'disabled'}) + + assert checks == () + + +def test_make_check_flags_with_checks_returns_flags(): + flags = module._make_check_flags(('foo', 'bar')) + + assert flags == ('--foo-only', '--bar-only') + + +def test_make_check_flags_with_default_checks_returns_no_flags(): + flags = module._make_check_flags(module.DEFAULT_CHECKS) + + assert flags == () + + def test_check_archives_should_call_attic_with_parameters(): + consistency_config = flexmock() + flexmock(module).should_receive('_parse_checks').and_return(flexmock()) + flexmock(module).should_receive('_make_check_flags').and_return(()) stdout = flexmock() insert_subprocess_mock( ('attic', 'check', 'repo'), @@ -127,10 +172,14 @@ def test_check_archives_should_call_attic_with_parameters(): module.check_archives( verbose=False, repository='repo', + consistency_config=consistency_config, ) def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters(): + consistency_config = flexmock() + flexmock(module).should_receive('_parse_checks').and_return(flexmock()) + flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( ('attic', 'check', 'repo', '--verbose'), stdout=None, @@ -141,4 +190,19 @@ def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters() module.check_archives( verbose=True, repository='repo', + consistency_config=consistency_config, ) + + +def test_check_archives_without_any_checks_should_bail(): + consistency_config = flexmock() + flexmock(module).should_receive('_parse_checks').and_return(()) + insert_subprocess_never() + + module.check_archives( + verbose=False, + repository='repo', + consistency_config=consistency_config, + ) + + diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py index 0576dc8..1c9fcb6 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/atticmatic/tests/unit/test_config.py @@ -41,19 +41,55 @@ def test_validate_configuration_format_with_valid_config_should_not_raise(): module.validate_configuration_format(parser, config_format) -def test_validate_configuration_format_with_missing_section_should_raise(): +def test_validate_configuration_format_with_missing_required_section_should_raise(): parser = flexmock() parser.should_receive('sections').and_return(('section',)) config_format = ( - module.Section_format('section', options=()), - module.Section_format('missing', options=()), + module.Section_format( + 'section', + options=( + module.Config_option('stuff', str, required=True), + ), + ), + # At least one option in this section is required, so the section is required. + module.Section_format( + 'missing', + options=( + module.Config_option('such', str, required=False), + module.Config_option('things', str, required=True), + ), + ), ) with assert_raises(ValueError): module.validate_configuration_format(parser, config_format) -def test_validate_configuration_format_with_extra_section_should_raise(): +def test_validate_configuration_format_with_missing_optional_section_should_not_raise(): + parser = flexmock() + parser.should_receive('sections').and_return(('section',)) + parser.should_receive('options').with_args('section').and_return(('stuff',)) + config_format = ( + module.Section_format( + 'section', + options=( + module.Config_option('stuff', str, required=True), + ), + ), + # No options in the section are required, so the section is optional. + module.Section_format( + 'missing', + options=( + module.Config_option('such', str, required=False), + module.Config_option('things', str, required=False), + ), + ), + ) + + module.validate_configuration_format(parser, config_format) + + +def test_validate_configuration_format_with_unknown_section_should_raise(): parser = flexmock() parser.should_receive('sections').and_return(('section', 'extra')) config_format = ( @@ -139,6 +175,26 @@ def test_parse_section_options_should_return_section_options(): ) +def test_parse_section_options_for_missing_section_should_return_empty_dict(): + parser = flexmock() + parser.should_receive('get').never() + parser.should_receive('getint').never() + parser.should_receive('has_option').with_args('section', 'foo').and_return(False) + parser.should_receive('has_option').with_args('section', 'bar').and_return(False) + + section_format = module.Section_format( + 'section', + ( + module.Config_option('foo', str, required=False), + module.Config_option('bar', int, required=False), + ), + ) + + config = module.parse_section_options(parser, section_format) + + assert config == OrderedDict() + + def insert_mock_parser(): parser = flexmock() parser.should_receive('readfp') @@ -154,13 +210,13 @@ def test_parse_configuration_should_return_section_configs(): mock_module.should_receive('validate_configuration_format').with_args( parser, module.CONFIG_FORMAT, ).once() - mock_section_configs = (flexmock(), flexmock()) + mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT) for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs): mock_module.should_receive('parse_section_options').with_args( parser, section_format, ).and_return(section_config).once() - section_configs = module.parse_configuration('filename') + parsed_config = module.parse_configuration('filename') - assert section_configs == mock_section_configs + assert parsed_config == module.Parsed_config(*mock_section_configs) diff --git a/sample/config b/sample/config index aa2acf3..b13afbe 100644 --- a/sample/config +++ b/sample/config @@ -6,8 +6,8 @@ source_directories: /home /etc repository: user@backupserver:sourcehostname.attic [retention] -# Retention policy for how many backups to keep in each category. -# See https://attic-backup.org/usage.html#attic-prune for details. +# Retention policy for how many backups to keep in each category. See +# https://attic-backup.org/usage.html#attic-prune for details. #keep_within: 3h #keep_hourly: 24 keep_daily: 7 @@ -15,3 +15,9 @@ keep_weekly: 4 keep_monthly: 6 keep_yearly: 1 #prefix: sourcehostname + +[consistency] +# Space-separated list of consistency checks to run: "repository", "archives", +# or both. Defaults to both. Set to "disabled" to disable all consistency +# checks. See https://attic-backup.org/usage.html#attic-check for details. +checks: repository archives diff --git a/setup.py b/setup.py index a771454..6fcd51d 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='atticmatic', - version='0.0.5', + version='0.0.6', description='A wrapper script for Attic backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', From c8f1af635f3660b4c6bfc7648437e8b455bbf812 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 10 May 2015 22:00:34 -0700 Subject: [PATCH 047/189] Added tag 0.0.6 for changeset 7ea93ca83f42 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index f3659ea..875f102 100644 --- a/.hgtags +++ b/.hgtags @@ -8,3 +8,4 @@ aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5 569aef47a9b25c55b13753f94706f5d330219995 0.0.5 569aef47a9b25c55b13753f94706f5d330219995 0.0.5 a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5 +7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6 From c3613e0637fb9ecd4e41dd2b8d529956d8af0022 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 10 May 2015 22:06:48 -0700 Subject: [PATCH 048/189] Adding some explanitory text about consistency checks to README example. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ee9cb72..771ca53 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Here's an example config file: keep_monthly: 6 [consistency] + # Consistency checks to run, or "disabled" to prevent checks. checks: repository archives Additionally, exclude patterns can be specified in a separate excludes config From d9125451f5987b1751aba582491f9063d1496f8d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 14 Jun 2015 11:00:46 -0700 Subject: [PATCH 049/189] Improved mocking of Python builtins in unit tests. --- NEWS | 4 ++++ atticmatic/config.py | 2 +- atticmatic/tests/builtins.py | 11 +++++++++++ atticmatic/tests/unit/test_attic.py | 9 ++++----- atticmatic/tests/unit/test_config.py | 5 ++--- 5 files changed, 22 insertions(+), 9 deletions(-) create mode 100644 atticmatic/tests/builtins.py diff --git a/NEWS b/NEWS index 5dbc3bc..302227f 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.0.7-dev + + * Improved mocking of Python builtins in unit tests. + 0.0.6 * New configuration section for customizing which Attic consistency checks run, if any. diff --git a/atticmatic/config.py b/atticmatic/config.py index 8254c88..7474f05 100644 --- a/atticmatic/config.py +++ b/atticmatic/config.py @@ -143,7 +143,7 @@ def parse_configuration(config_filename): Raise IOError if the file cannot be read, or ValueError if the format is not as expected. ''' parser = ConfigParser() - parser.readfp(open(config_filename)) + parser.read(config_filename) validate_configuration_format(parser, CONFIG_FORMAT) diff --git a/atticmatic/tests/builtins.py b/atticmatic/tests/builtins.py new file mode 100644 index 0000000..ff48f93 --- /dev/null +++ b/atticmatic/tests/builtins.py @@ -0,0 +1,11 @@ +from flexmock import flexmock +import sys + + +def builtins_mock(): + try: + # Python 2 + return flexmock(sys.modules['__builtin__']) + except KeyError: + # Python 3 + return flexmock(sys.modules['builtins']) diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py index 26c6299..ab52850 100644 --- a/atticmatic/tests/unit/test_attic.py +++ b/atticmatic/tests/unit/test_attic.py @@ -3,6 +3,7 @@ from collections import OrderedDict from flexmock import flexmock from atticmatic import attic as module +from atticmatic.tests.builtins import builtins_mock def insert_subprocess_mock(check_call_command, **kwargs): @@ -18,7 +19,7 @@ def insert_subprocess_never(): def insert_platform_mock(): - flexmock(module).platform = flexmock().should_receive('node').and_return('host').mock + flexmock(module.platform).should_receive('node').and_return('host') def insert_datetime_mock(): @@ -166,8 +167,8 @@ def test_check_archives_should_call_attic_with_parameters(): ) insert_platform_mock() insert_datetime_mock() - flexmock(module).open = lambda filename, mode: stdout - flexmock(module).os = flexmock().should_receive('devnull').mock + builtins_mock().should_receive('open').and_return(stdout) + flexmock(module.os).should_receive('devnull') module.check_archives( verbose=False, @@ -204,5 +205,3 @@ def test_check_archives_without_any_checks_should_bail(): repository='repo', consistency_config=consistency_config, ) - - diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py index 1c9fcb6..02f1f52 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/atticmatic/tests/unit/test_config.py @@ -197,9 +197,8 @@ def test_parse_section_options_for_missing_section_should_return_empty_dict(): def insert_mock_parser(): parser = flexmock() - parser.should_receive('readfp') - flexmock(module).open = lambda filename: None - flexmock(module).ConfigParser = parser + parser.should_receive('read') + module.ConfigParser = lambda: parser return parser From 5bf3a4875c596574bf57a476108eb19c881b88ec Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 17 Jul 2015 21:58:50 -0700 Subject: [PATCH 050/189] Flag for multiple levels of verbosity: some, and lots. --- NEWS | 3 +- README.md | 8 +- atticmatic/attic.py | 32 +++++-- atticmatic/command.py | 12 +-- atticmatic/tests/integration/test_command.py | 10 +- atticmatic/tests/unit/test_attic.py | 99 ++++++++++++++------ atticmatic/verbosity.py | 2 + setup.py | 2 +- 8 files changed, 113 insertions(+), 55 deletions(-) create mode 100644 atticmatic/verbosity.py diff --git a/NEWS b/NEWS index 302227f..f46598a 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ -0.0.7-dev +0.0.7 + * Flag for multiple levels of verbosity: some, and lots. * Improved mocking of Python builtins in unit tests. 0.0.6 diff --git a/README.md b/README.md index 771ca53..67b4bc1 100644 --- a/README.md +++ b/README.md @@ -78,9 +78,13 @@ and check backups for consistency problems due to things like file damage. By default, the backup will proceed silently except in the case of errors. But if you'd like to to get additional information about the progress of the -backup as it proceeds, use the verbose option instead: +backup as it proceeds, use the verbosity option: - atticmattic --verbose + atticmattic --verbosity 1 + +Or, for even more progress spew: + + atticmattic --verbosity 2 If you'd like to see the available command-line arguments, view the help: diff --git a/atticmatic/attic.py b/atticmatic/attic.py index fc83997..5a0a4b7 100644 --- a/atticmatic/attic.py +++ b/atticmatic/attic.py @@ -3,13 +3,19 @@ import os import platform import subprocess +from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS -def create_archive(excludes_filename, verbose, source_directories, repository): + +def create_archive(excludes_filename, verbosity, source_directories, repository): ''' Given an excludes filename, a vebosity flag, a space-separated list of source directories, and a local or remote repository path, create an attic archive. ''' sources = tuple(source_directories.split(' ')) + verbosity_flags = { + VERBOSITY_SOME: ('--stats',), + VERBOSITY_LOTS: ('--verbose', '--stats'), + }.get(verbosity, ()) command = ( 'attic', 'create', @@ -19,9 +25,7 @@ def create_archive(excludes_filename, verbose, source_directories, repository): hostname=platform.node(), timestamp=datetime.now().isoformat(), ), - ) + sources + ( - ('--verbose', '--stats') if verbose else () - ) + ) + sources + verbosity_flags subprocess.check_call(command) @@ -48,11 +52,16 @@ def _make_prune_flags(retention_config): ) -def prune_archives(verbose, repository, retention_config): +def prune_archives(verbosity, repository, retention_config): ''' Given a verbosity flag, a local or remote repository path, and a retention config dict, prune attic archives according the the retention policy specified in that configuration. ''' + verbosity_flags = { + VERBOSITY_SOME: ('--stats',), + VERBOSITY_LOTS: ('--verbose', '--stats'), + }.get(verbosity, ()) + command = ( 'attic', 'prune', repository, @@ -60,7 +69,7 @@ def prune_archives(verbose, repository, retention_config): element for pair in _make_prune_flags(retention_config) for element in pair - ) + (('--verbose',) if verbose else ()) + ) + verbosity_flags subprocess.check_call(command) @@ -114,7 +123,7 @@ def _make_check_flags(checks): ) -def check_archives(verbose, repository, consistency_config): +def check_archives(verbosity, repository, consistency_config): ''' Given a verbosity flag, a local or remote repository path, and a consistency config dict, check the contained attic archives for consistency. @@ -125,12 +134,17 @@ def check_archives(verbose, repository, consistency_config): if not checks: return + verbosity_flags = { + VERBOSITY_SOME: ('--verbose',), + VERBOSITY_LOTS: ('--verbose',), + }.get(verbosity, ()) + command = ( 'attic', 'check', repository, - ) + _make_check_flags(checks) + (('--verbose',) if verbose else ()) + ) + _make_check_flags(checks) + verbosity_flags # Attic's check command spews to stdout even without the verbose flag. Suppress it. - stdout = None if verbose else open(os.devnull, 'w') + stdout = None if verbosity_flags else open(os.devnull, 'w') subprocess.check_call(command, stdout=stdout) diff --git a/atticmatic/command.py b/atticmatic/command.py index 0a844b2..37f42be 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -29,9 +29,9 @@ def parse_arguments(*arguments): help='Excludes filename', ) parser.add_argument( - '-v', '--verbose', - action='store_true', - help='Display verbose progress information', + '-v', '--verbosity', + type=int, + help='Display verbose progress (1 for some, 2 for lots)', ) return parser.parse_args(arguments) @@ -43,9 +43,9 @@ def main(): config = parse_configuration(args.config_filename) repository = config.location['repository'] - create_archive(args.excludes_filename, args.verbose, **config.location) - prune_archives(args.verbose, repository, config.retention) - check_archives(args.verbose, repository, config.consistency) + create_archive(args.excludes_filename, args.verbosity, **config.location) + prune_archives(args.verbosity, repository, config.retention) + check_archives(args.verbosity, repository, config.consistency) except (ValueError, IOError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/atticmatic/tests/integration/test_command.py b/atticmatic/tests/integration/test_command.py index 3a5e794..9b75871 100644 --- a/atticmatic/tests/integration/test_command.py +++ b/atticmatic/tests/integration/test_command.py @@ -10,7 +10,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME - assert parser.verbose == False + assert parser.verbosity == None def test_parse_arguments_with_filename_arguments_overrides_defaults(): @@ -18,15 +18,15 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults(): assert parser.config_filename == 'myconfig' assert parser.excludes_filename == 'myexcludes' - assert parser.verbose == False + assert parser.verbosity == None -def test_parse_arguments_with_verbose_flag_overrides_default(): - parser = module.parse_arguments('--verbose') +def test_parse_arguments_with_verbosity_flag_overrides_default(): + parser = module.parse_arguments('--verbosity', '1') assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME - assert parser.verbose == True + assert parser.verbosity == 1 def test_parse_arguments_with_invalid_arguments_exits(): diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/test_attic.py index ab52850..13be67b 100644 --- a/atticmatic/tests/unit/test_attic.py +++ b/atticmatic/tests/unit/test_attic.py @@ -4,6 +4,7 @@ from flexmock import flexmock from atticmatic import attic as module from atticmatic.tests.builtins import builtins_mock +from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS def insert_subprocess_mock(check_call_command, **kwargs): @@ -28,34 +29,43 @@ def insert_datetime_mock(): ).mock +CREATE_COMMAND = ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar') + + def test_create_archive_should_call_attic_with_parameters(): - insert_subprocess_mock( - ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'), - ) + insert_subprocess_mock(CREATE_COMMAND) insert_platform_mock() insert_datetime_mock() module.create_archive( excludes_filename='excludes', - verbose=False, + verbosity=None, source_directories='foo bar', repository='repo', ) -def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters(): - insert_subprocess_mock( - ( - 'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar', - '--verbose', '--stats', - ), - ) +def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter(): + insert_subprocess_mock(CREATE_COMMAND + ('--stats',)) insert_platform_mock() insert_datetime_mock() module.create_archive( excludes_filename='excludes', - verbose=True, + verbosity=VERBOSITY_SOME, + source_directories='foo bar', + repository='repo', + ) + + +def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_parameter(): + insert_subprocess_mock(CREATE_COMMAND + ('--verbose', '--stats')) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + excludes_filename='excludes', + verbosity=VERBOSITY_LOTS, source_directories='foo bar', repository='repo', ) @@ -82,40 +92,49 @@ def test_make_prune_flags_should_return_flags_from_config(): assert tuple(result) == BASE_PRUNE_FLAGS +PRUNE_COMMAND = ( + 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', +) + + def test_prune_archives_should_call_attic_with_parameters(): retention_config = flexmock() flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) - insert_subprocess_mock( - ( - 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', - '3', - ), - ) + insert_subprocess_mock(PRUNE_COMMAND) module.prune_archives( - verbose=False, + verbosity=None, repository='repo', retention_config=retention_config, ) -def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters(): +def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_parameter(): retention_config = flexmock() flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) - insert_subprocess_mock( - ( - 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', - '3', '--verbose', - ), - ) + insert_subprocess_mock(PRUNE_COMMAND + ('--stats',)) module.prune_archives( repository='repo', - verbose=True, + verbosity=VERBOSITY_SOME, + retention_config=retention_config, + ) + + +def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter(): + retention_config = flexmock() + flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( + BASE_PRUNE_FLAGS, + ) + insert_subprocess_mock(PRUNE_COMMAND + ('--verbose', '--stats',)) + + module.prune_archives( + repository='repo', + verbosity=VERBOSITY_LOTS, retention_config=retention_config, ) @@ -171,13 +190,13 @@ def test_check_archives_should_call_attic_with_parameters(): flexmock(module.os).should_receive('devnull') module.check_archives( - verbose=False, + verbosity=None, repository='repo', consistency_config=consistency_config, ) -def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters(): +def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter(): consistency_config = flexmock() flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_make_check_flags').and_return(()) @@ -189,7 +208,25 @@ def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters() insert_datetime_mock() module.check_archives( - verbose=True, + verbosity=VERBOSITY_SOME, + repository='repo', + consistency_config=consistency_config, + ) + + +def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter(): + consistency_config = flexmock() + flexmock(module).should_receive('_parse_checks').and_return(flexmock()) + flexmock(module).should_receive('_make_check_flags').and_return(()) + insert_subprocess_mock( + ('attic', 'check', 'repo', '--verbose'), + stdout=None, + ) + insert_platform_mock() + insert_datetime_mock() + + module.check_archives( + verbosity=VERBOSITY_LOTS, repository='repo', consistency_config=consistency_config, ) @@ -201,7 +238,7 @@ def test_check_archives_without_any_checks_should_bail(): insert_subprocess_never() module.check_archives( - verbose=False, + verbosity=None, repository='repo', consistency_config=consistency_config, ) diff --git a/atticmatic/verbosity.py b/atticmatic/verbosity.py new file mode 100644 index 0000000..06dfc4c --- /dev/null +++ b/atticmatic/verbosity.py @@ -0,0 +1,2 @@ +VERBOSITY_SOME = 1 +VERBOSITY_LOTS = 2 diff --git a/setup.py b/setup.py index 6fcd51d..2a64c55 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='atticmatic', - version='0.0.6', + version='0.0.7', description='A wrapper script for Attic backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', From 52d5240fa0e61ca1d9119ef59b24078ecde3bd6e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 17 Jul 2015 21:58:58 -0700 Subject: [PATCH 051/189] Added tag 0.0.7 for changeset cf4c7065f071 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 875f102..f43c3a7 100644 --- a/.hgtags +++ b/.hgtags @@ -9,3 +9,4 @@ aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5 569aef47a9b25c55b13753f94706f5d330219995 0.0.5 a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5 7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6 +cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7 From 7097ed67a6b9a3035112dade86ce5cbcd4a778e6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 18 Jul 2015 18:35:29 -0700 Subject: [PATCH 052/189] New "borgmatic" command to support Borg backup software, a fork of Attic. --- NEWS | 4 ++ README.md | 40 +++++++++++------ atticmatic/backends/__init__.py | 0 atticmatic/backends/attic.py | 12 +++++ atticmatic/backends/borg.py | 13 ++++++ atticmatic/{attic.py => backends/shared.py} | 45 +++++++++++-------- atticmatic/command.py | 40 ++++++++++++----- atticmatic/tests/integration/test_command.py | 21 +++++---- atticmatic/tests/unit/backends/__init__.py | 0 .../test_shared.py} | 12 ++++- atticmatic/tests/unit/test_command.py | 33 ++++++++++++++ sample/config | 2 +- setup.py | 11 +++-- 13 files changed, 175 insertions(+), 58 deletions(-) create mode 100644 atticmatic/backends/__init__.py create mode 100644 atticmatic/backends/attic.py create mode 100644 atticmatic/backends/borg.py rename atticmatic/{attic.py => backends/shared.py} (74%) create mode 100644 atticmatic/tests/unit/backends/__init__.py rename atticmatic/tests/unit/{test_attic.py => backends/test_shared.py} (95%) create mode 100644 atticmatic/tests/unit/test_command.py diff --git a/NEWS b/NEWS index f46598a..f8dd0bf 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.1.0 + + * New "borgmatic" command to support Borg backup software, a fork of Attic. + 0.0.7 * Flag for multiple levels of verbosity: some, and lots. diff --git a/README.md b/README.md index 67b4bc1..fe26676 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,13 @@ save_as: atticmatic/index.html ## Overview -atticmatic is a simple Python wrapper script for the [Attic backup -software](https://attic-backup.org/) that initiates a backup, prunes any old -backups according to a retention policy, and validates backups for -consistency. The script supports specifying your settings in a declarative -configuration file rather than having to put them all on the command-line, and -handles common errors. +atticmatic is a simple Python wrapper script for the +[Attic](https://attic-backup.org/) and +[Borg](https://borgbackup.github.io/borgbackup/) backup software that +initiates a backup, prunes any old backups according to a retention policy, +and validates backups for consistency. The script supports specifying your +settings in a declarative configuration file rather than having to put them +all on the command-line, and handles common errors. Here's an example config file: @@ -17,7 +18,7 @@ Here's an example config file: # Space-separated list of source directories to backup. source_directories: /home /etc - # Path to local or remote Attic repository. + # Path to local or remote backup repository. repository: user@backupserver:sourcehostname.attic [retention] @@ -41,14 +42,14 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on ## Setup -To get up and running with Attic, follow the [Attic Quick -Start](https://attic-backup.org/quickstart.html) guide to create an Attic +To get up and running, follow the [Attic Quick +Start](https://attic-backup.org/quickstart.html) or the [Borg Quick +Start](https://borgbackup.github.io/borgbackup/quickstart.html) to create a repository on a local or remote host. Note that if you plan to run atticmatic on a schedule with cron, and you encrypt your attic repository with a passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE` -environment variable. See [attic's repository encryption -documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for -more info. +environment variable. See the repository encryption section of the Quick Start +for more info. If the repository is on a remote host, make sure that your local root user has key-based ssh access to the desired user account on the remote host. @@ -57,13 +58,19 @@ To install atticmatic, run the following command to download and install it: sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic -Then copy the following configuration files: +If you are using Attic, copy the following configuration files: sudo cp sample/atticmatic.cron /etc/cron.d/atticmatic sudo mkdir /etc/atticmatic/ sudo cp sample/config sample/excludes /etc/atticmatic/ -Lastly, modify those files with your desired configuration. +If you are using Borg, copy the files like this instead: + + sudo cp sample/atticmatic.cron /etc/cron.d/borgmatic + sudo mkdir /etc/borgmatic/ + sudo cp sample/config sample/excludes /etc/borgmatic/ + +Lastly, modify the /etc files with your desired configuration. ## Usage @@ -73,6 +80,11 @@ arguments: atticmatic +Or, if you're using Borg, use this command instead to make use of the Borg +backend: + + borgmatic + This will also prune any old backups as per the configured retention policy, and check backups for consistency problems due to things like file damage. diff --git a/atticmatic/backends/__init__.py b/atticmatic/backends/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/atticmatic/backends/attic.py b/atticmatic/backends/attic.py new file mode 100644 index 0000000..a99c323 --- /dev/null +++ b/atticmatic/backends/attic.py @@ -0,0 +1,12 @@ +from functools import partial + +from atticmatic.backends import shared + +# An atticmatic backend that supports Attic for actually handling backups. + +COMMAND = 'attic' + + +create_archive = partial(shared.create_archive, command=COMMAND) +prune_archives = partial(shared.prune_archives, command=COMMAND) +check_archives = partial(shared.check_archives, command=COMMAND) diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py new file mode 100644 index 0000000..96b5bff --- /dev/null +++ b/atticmatic/backends/borg.py @@ -0,0 +1,13 @@ +from functools import partial + +from atticmatic.backends import shared + +# An atticmatic backend that supports Borg for actually handling backups. + +COMMAND = 'borg' + + +create_archive = partial(shared.create_archive, command=COMMAND) +prune_archives = partial(shared.prune_archives, command=COMMAND) +check_archives = partial(shared.check_archives, command=COMMAND) + diff --git a/atticmatic/attic.py b/atticmatic/backends/shared.py similarity index 74% rename from atticmatic/attic.py rename to atticmatic/backends/shared.py index 5a0a4b7..eb167b5 100644 --- a/atticmatic/attic.py +++ b/atticmatic/backends/shared.py @@ -6,10 +6,16 @@ import subprocess from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS -def create_archive(excludes_filename, verbosity, source_directories, repository): +# Common backend functionality shared by Attic and Borg. As the two backup +# commands diverge, these shared functions will likely need to be replaced +# with non-shared functions within atticmatic.backends.attic and +# atticmatic.backends.borg. + + +def create_archive(excludes_filename, verbosity, source_directories, repository, command): ''' - Given an excludes filename, a vebosity flag, a space-separated list of source directories, and - a local or remote repository path, create an attic archive. + Given an excludes filename, a vebosity flag, a space-separated list of source directories, a + local or remote repository path, and a command to run, create an attic archive. ''' sources = tuple(source_directories.split(' ')) verbosity_flags = { @@ -17,8 +23,8 @@ def create_archive(excludes_filename, verbosity, source_directories, repository) VERBOSITY_LOTS: ('--verbose', '--stats'), }.get(verbosity, ()) - command = ( - 'attic', 'create', + full_command = ( + command, 'create', '--exclude-from', excludes_filename, '{repo}::{hostname}-{timestamp}'.format( repo=repository, @@ -27,7 +33,7 @@ def create_archive(excludes_filename, verbosity, source_directories, repository) ), ) + sources + verbosity_flags - subprocess.check_call(command) + subprocess.check_call(full_command) def _make_prune_flags(retention_config): @@ -52,18 +58,19 @@ def _make_prune_flags(retention_config): ) -def prune_archives(verbosity, repository, retention_config): +def prune_archives(verbosity, repository, retention_config, command): ''' - Given a verbosity flag, a local or remote repository path, and a retention config dict, prune - attic archives according the the retention policy specified in that configuration. + Given a verbosity flag, a local or remote repository path, a retention config dict, and a + command to run, prune attic archives according the the retention policy specified in that + configuration. ''' verbosity_flags = { VERBOSITY_SOME: ('--stats',), VERBOSITY_LOTS: ('--verbose', '--stats'), }.get(verbosity, ()) - command = ( - 'attic', 'prune', + full_command = ( + command, 'prune', repository, ) + tuple( element @@ -71,7 +78,7 @@ def prune_archives(verbosity, repository, retention_config): for element in pair ) + verbosity_flags - subprocess.check_call(command) + subprocess.check_call(full_command) DEFAULT_CHECKS = ('repository', 'archives') @@ -123,10 +130,10 @@ def _make_check_flags(checks): ) -def check_archives(verbosity, repository, consistency_config): +def check_archives(verbosity, repository, consistency_config, command): ''' - Given a verbosity flag, a local or remote repository path, and a consistency config dict, check - the contained attic archives for consistency. + Given a verbosity flag, a local or remote repository path, a consistency config dict, and a + command to run, check the contained attic archives for consistency. If there are no consistency checks to run, skip running them. ''' @@ -139,12 +146,12 @@ def check_archives(verbosity, repository, consistency_config): VERBOSITY_LOTS: ('--verbose',), }.get(verbosity, ()) - command = ( - 'attic', 'check', + full_command = ( + command, 'check', repository, ) + _make_check_flags(checks) + verbosity_flags - # Attic's check command spews to stdout even without the verbose flag. Suppress it. + # The check command spews to stdout even without the verbose flag. Suppress it. stdout = None if verbosity_flags else open(os.devnull, 'w') - subprocess.check_call(command, stdout=stdout) + subprocess.check_call(full_command, stdout=stdout) diff --git a/atticmatic/command.py b/atticmatic/command.py index 37f42be..4578c1e 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -1,31 +1,34 @@ from __future__ import print_function from argparse import ArgumentParser +from importlib import import_module +import os from subprocess import CalledProcessError import sys -from atticmatic.attic import check_archives, create_archive, prune_archives from atticmatic.config import parse_configuration -DEFAULT_CONFIG_FILENAME = '/etc/atticmatic/config' -DEFAULT_EXCLUDES_FILENAME = '/etc/atticmatic/excludes' +DEFAULT_CONFIG_FILENAME_PATTERN = '/etc/{}/config' +DEFAULT_EXCLUDES_FILENAME_PATTERN = '/etc/{}/excludes' -def parse_arguments(*arguments): +def parse_arguments(command_name, *arguments): ''' - Parse the given command-line arguments and return them as an ArgumentParser instance. + Given the name of the command with which this script was invoked and command-line arguments, + parse the arguments and return them as an ArgumentParser instance. Use the command name to + determine the default configuration and excludes paths. ''' parser = ArgumentParser() parser.add_argument( '-c', '--config', dest='config_filename', - default=DEFAULT_CONFIG_FILENAME, + default=DEFAULT_CONFIG_FILENAME_PATTERN.format(command_name), help='Configuration filename', ) parser.add_argument( '--excludes', dest='excludes_filename', - default=DEFAULT_EXCLUDES_FILENAME, + default=DEFAULT_EXCLUDES_FILENAME_PATTERN.format(command_name), help='Excludes filename', ) parser.add_argument( @@ -37,15 +40,30 @@ def parse_arguments(*arguments): return parser.parse_args(arguments) +def load_backend(command_name): + ''' + Given the name of the command with which this script was invoked, return the corresponding + backend module responsible for actually dealing with backups. + ''' + backend_name = { + 'atticmatic': 'attic', + 'borgmatic': 'borg', + }.get(command_name, 'attic') + + return import_module('atticmatic.backends.{}'.format(backend_name)) + + def main(): try: - args = parse_arguments(*sys.argv[1:]) + command_name = os.path.basename(sys.argv[0]) + args = parse_arguments(command_name, *sys.argv[1:]) config = parse_configuration(args.config_filename) repository = config.location['repository'] + backend = load_backend(command_name) - create_archive(args.excludes_filename, args.verbosity, **config.location) - prune_archives(args.verbosity, repository, config.retention) - check_archives(args.verbosity, repository, config.consistency) + backend.create_archive(args.excludes_filename, args.verbosity, **config.location) + backend.prune_archives(args.verbosity, repository, config.retention) + backend.check_archives(args.verbosity, repository, config.consistency) except (ValueError, IOError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/atticmatic/tests/integration/test_command.py b/atticmatic/tests/integration/test_command.py index 9b75871..fd72595 100644 --- a/atticmatic/tests/integration/test_command.py +++ b/atticmatic/tests/integration/test_command.py @@ -5,16 +5,19 @@ from nose.tools import assert_raises from atticmatic import command as module -def test_parse_arguments_with_no_arguments_uses_defaults(): - parser = module.parse_arguments() +COMMAND_NAME = 'foomatic' - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME - assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME + +def test_parse_arguments_with_no_arguments_uses_defaults(): + parser = module.parse_arguments(COMMAND_NAME) + + assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME) + assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME_PATTERN.format(COMMAND_NAME) assert parser.verbosity == None def test_parse_arguments_with_filename_arguments_overrides_defaults(): - parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes') + parser = module.parse_arguments(COMMAND_NAME, '--config', 'myconfig', '--excludes', 'myexcludes') assert parser.config_filename == 'myconfig' assert parser.excludes_filename == 'myexcludes' @@ -22,10 +25,10 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults(): def test_parse_arguments_with_verbosity_flag_overrides_default(): - parser = module.parse_arguments('--verbosity', '1') + parser = module.parse_arguments(COMMAND_NAME, '--verbosity', '1') - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME - assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME + assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME) + assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME_PATTERN.format(COMMAND_NAME) assert parser.verbosity == 1 @@ -35,6 +38,6 @@ def test_parse_arguments_with_invalid_arguments_exits(): try: with assert_raises(SystemExit): - module.parse_arguments('--posix-me-harder') + module.parse_arguments(COMMAND_NAME, '--posix-me-harder') finally: sys.stderr = original_stderr diff --git a/atticmatic/tests/unit/backends/__init__.py b/atticmatic/tests/unit/backends/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/atticmatic/tests/unit/test_attic.py b/atticmatic/tests/unit/backends/test_shared.py similarity index 95% rename from atticmatic/tests/unit/test_attic.py rename to atticmatic/tests/unit/backends/test_shared.py index 13be67b..c779c36 100644 --- a/atticmatic/tests/unit/test_attic.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -2,7 +2,7 @@ from collections import OrderedDict from flexmock import flexmock -from atticmatic import attic as module +from atticmatic.backends import shared as module from atticmatic.tests.builtins import builtins_mock from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS @@ -42,6 +42,7 @@ def test_create_archive_should_call_attic_with_parameters(): verbosity=None, source_directories='foo bar', repository='repo', + command='attic', ) @@ -55,6 +56,7 @@ def test_create_archive_with_verbosity_some_should_call_attic_with_stats_paramet verbosity=VERBOSITY_SOME, source_directories='foo bar', repository='repo', + command='attic', ) @@ -68,6 +70,7 @@ def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_param verbosity=VERBOSITY_LOTS, source_directories='foo bar', repository='repo', + command='attic', ) @@ -108,6 +111,7 @@ def test_prune_archives_should_call_attic_with_parameters(): verbosity=None, repository='repo', retention_config=retention_config, + command='attic', ) @@ -122,6 +126,7 @@ def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_paramet repository='repo', verbosity=VERBOSITY_SOME, retention_config=retention_config, + command='attic', ) @@ -136,6 +141,7 @@ def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_param repository='repo', verbosity=VERBOSITY_LOTS, retention_config=retention_config, + command='attic', ) @@ -193,6 +199,7 @@ def test_check_archives_should_call_attic_with_parameters(): verbosity=None, repository='repo', consistency_config=consistency_config, + command='attic', ) @@ -211,6 +218,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param verbosity=VERBOSITY_SOME, repository='repo', consistency_config=consistency_config, + command='attic', ) @@ -229,6 +237,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param verbosity=VERBOSITY_LOTS, repository='repo', consistency_config=consistency_config, + command='attic', ) @@ -241,4 +250,5 @@ def test_check_archives_without_any_checks_should_bail(): verbosity=None, repository='repo', consistency_config=consistency_config, + command='attic', ) diff --git a/atticmatic/tests/unit/test_command.py b/atticmatic/tests/unit/test_command.py new file mode 100644 index 0000000..6a3cdb1 --- /dev/null +++ b/atticmatic/tests/unit/test_command.py @@ -0,0 +1,33 @@ +from flexmock import flexmock + +from atticmatic import command as module + + +def test_load_backend_with_atticmatic_command_should_return_attic_backend(): + backend = flexmock() + ( + flexmock(module).should_receive('import_module').with_args('atticmatic.backends.attic') + .and_return(backend).once() + ) + + assert module.load_backend('atticmatic') == backend + + +def test_load_backend_with_unknown_command_should_return_attic_backend(): + backend = flexmock() + ( + flexmock(module).should_receive('import_module').with_args('atticmatic.backends.attic') + .and_return(backend).once() + ) + + assert module.load_backend('unknownmatic') == backend + + +def test_load_backend_with_borgmatic_command_should_return_borg_backend(): + backend = flexmock() + ( + flexmock(module).should_receive('import_module').with_args('atticmatic.backends.borg') + .and_return(backend).once() + ) + + assert module.load_backend('borgmatic') == backend diff --git a/sample/config b/sample/config index b13afbe..6ae08b4 100644 --- a/sample/config +++ b/sample/config @@ -2,7 +2,7 @@ # Space-separated list of source directories to backup. source_directories: /home /etc -# Path to local or remote Attic repository. +# Path to local or remote repository. repository: user@backupserver:sourcehostname.attic [retention] diff --git a/setup.py b/setup.py index 2a64c55..5736ccc 100644 --- a/setup.py +++ b/setup.py @@ -2,12 +2,17 @@ from setuptools import setup, find_packages setup( name='atticmatic', - version='0.0.7', - description='A wrapper script for Attic backup software that creates and prunes backups', + version='0.1.0', + description='A wrapper script for Attic/Borg backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', packages=find_packages(), - entry_points={'console_scripts': ['atticmatic = atticmatic.command:main']}, + entry_points={ + 'console_scripts': [ + 'atticmatic = atticmatic.command:main', + 'borgmatic = atticmatic.command:main', + ] + }, tests_require=( 'flexmock', 'nose', From d25db4cd0dda7677f93ba26b324a8d16d87d62af Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 18 Jul 2015 18:39:33 -0700 Subject: [PATCH 053/189] Added tag 0.1.0 for changeset 38d72677343f --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index f43c3a7..118bf80 100644 --- a/.hgtags +++ b/.hgtags @@ -10,3 +10,4 @@ aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5 a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5 7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6 cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7 +38d72677343f0a5d6845f4ac50d6778397083d45 0.1.0 From 6b0aa13856835e1a415c01fe66df8faa533939a1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 18 Jul 2015 18:44:11 -0700 Subject: [PATCH 054/189] Adding borgmatic cron example. --- NEWS | 4 ++++ README.md | 2 +- sample/borgmatic.cron | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 sample/borgmatic.cron diff --git a/NEWS b/NEWS index f8dd0bf..159c71c 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.1.1 + + * Adding borgmatic cron example, and updating documentation to refer to it. + 0.1.0 * New "borgmatic" command to support Borg backup software, a fork of Attic. diff --git a/README.md b/README.md index fe26676..69f18c0 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ If you are using Attic, copy the following configuration files: If you are using Borg, copy the files like this instead: - sudo cp sample/atticmatic.cron /etc/cron.d/borgmatic + sudo cp sample/borgmatic.cron /etc/cron.d/borgmatic sudo mkdir /etc/borgmatic/ sudo cp sample/config sample/excludes /etc/borgmatic/ diff --git a/sample/borgmatic.cron b/sample/borgmatic.cron new file mode 100644 index 0000000..2f6f912 --- /dev/null +++ b/sample/borgmatic.cron @@ -0,0 +1,3 @@ +# You can drop this file into /etc/cron.d/ to run borgmatic nightly. + +0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/borgmatic From ce6196a5c6f62607629d952614c511fceb67cdbd Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 18 Jul 2015 18:44:14 -0700 Subject: [PATCH 055/189] Added tag 0.1.1 for changeset ac5dfa01e9d1 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 118bf80..642f1c3 100644 --- a/.hgtags +++ b/.hgtags @@ -11,3 +11,4 @@ a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5 7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6 cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7 38d72677343f0a5d6845f4ac50d6778397083d45 0.1.0 +ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1 From 1f1c8fdabacdd2a8b896fa1219be5a6de86a0820 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 18 Jul 2015 23:48:58 -0700 Subject: [PATCH 056/189] Bumping version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5736ccc..7d535bc 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='atticmatic', - version='0.1.0', + version='0.1.1', description='A wrapper script for Attic/Borg backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', From 17ac63aae6e63a4ea3df55dca4cdd51f882271b2 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 18 Jul 2015 23:49:06 -0700 Subject: [PATCH 057/189] Added tag 0.1.1 for changeset 7b6c87dca7ea --- .hgtags | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.hgtags b/.hgtags index 642f1c3..cc49115 100644 --- a/.hgtags +++ b/.hgtags @@ -12,3 +12,5 @@ a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5 cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7 38d72677343f0a5d6845f4ac50d6778397083d45 0.1.0 ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1 +ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1 +7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1 From 52ab7cb881cbd9f3e3c43cb30cfe5fc5d04b9b5e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 21 Jul 2015 21:29:40 -0700 Subject: [PATCH 058/189] New issue tracker, linked from documentation. --- NEWS | 4 ++++ README.md | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 159c71c..5f6243c 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.1.2-dev + + * New issue tracker, linked from documentation. + 0.1.1 * Adding borgmatic cron example, and updating documentation to refer to it. diff --git a/README.md b/README.md index 69f18c0..b738c96 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,9 @@ This should make the client keep the connection alive while validating backups. -## Feedback +## Issues and feedback -Questions? Comments? Got a patch? Contact . +Got an issue or an idea for a feature enhancement? Check out the [atticmatic +issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues). + +Other questions or comments? Contact . From 38322a3f6fe71f9bbd0077cb628388ff36c98a6a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 26 Jul 2015 20:57:31 -0700 Subject: [PATCH 059/189] Linking to both Attic and Borg prune docs from sample config. --- sample/config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sample/config b/sample/config index 6ae08b4..4e09b04 100644 --- a/sample/config +++ b/sample/config @@ -7,7 +7,8 @@ repository: user@backupserver:sourcehostname.attic [retention] # Retention policy for how many backups to keep in each category. See -# https://attic-backup.org/usage.html#attic-prune for details. +# https://attic-backup.org/usage.html#attic-prune or +# https://borgbackup.github.io/borgbackup/usage.html#borg-prune for details. #keep_within: 3h #keep_hourly: 24 keep_daily: 7 From 58d33503a1bfbe4e44332d388a7b0dc63099e7dc Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 26 Jul 2015 21:06:03 -0700 Subject: [PATCH 060/189] As a convenience to new users, allow a missing default excludes file. --- NEWS | 3 +- atticmatic/backends/shared.py | 8 ++--- atticmatic/command.py | 7 +++-- atticmatic/tests/integration/test_command.py | 29 +++++++++++++++++++ atticmatic/tests/unit/backends/test_shared.py | 17 ++++++++++- 5 files changed, 56 insertions(+), 8 deletions(-) diff --git a/NEWS b/NEWS index 5f6243c..db51ca3 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ -0.1.2-dev +0.1.2 + * As a convenience to new users, allow a missing default excludes file. * New issue tracker, linked from documentation. 0.1.1 diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py index eb167b5..8abd064 100644 --- a/atticmatic/backends/shared.py +++ b/atticmatic/backends/shared.py @@ -14,10 +14,11 @@ from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS def create_archive(excludes_filename, verbosity, source_directories, repository, command): ''' - Given an excludes filename, a vebosity flag, a space-separated list of source directories, a - local or remote repository path, and a command to run, create an attic archive. + Given an excludes filename (or None), a vebosity flag, a space-separated list of source + directories, a local or remote repository path, and a command to run, create an attic archive. ''' sources = tuple(source_directories.split(' ')) + exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else () verbosity_flags = { VERBOSITY_SOME: ('--stats',), VERBOSITY_LOTS: ('--verbose', '--stats'), @@ -25,13 +26,12 @@ def create_archive(excludes_filename, verbosity, source_directories, repository, full_command = ( command, 'create', - '--exclude-from', excludes_filename, '{repo}::{hostname}-{timestamp}'.format( repo=repository, hostname=platform.node(), timestamp=datetime.now().isoformat(), ), - ) + sources + verbosity_flags + ) + sources + exclude_flags + verbosity_flags subprocess.check_call(full_command) diff --git a/atticmatic/command.py b/atticmatic/command.py index 4578c1e..13e7b43 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -18,17 +18,20 @@ def parse_arguments(command_name, *arguments): parse the arguments and return them as an ArgumentParser instance. Use the command name to determine the default configuration and excludes paths. ''' + config_filename_default = DEFAULT_CONFIG_FILENAME_PATTERN.format(command_name) + excludes_filename_default = DEFAULT_EXCLUDES_FILENAME_PATTERN.format(command_name) + parser = ArgumentParser() parser.add_argument( '-c', '--config', dest='config_filename', - default=DEFAULT_CONFIG_FILENAME_PATTERN.format(command_name), + default=config_filename_default, help='Configuration filename', ) parser.add_argument( '--excludes', dest='excludes_filename', - default=DEFAULT_EXCLUDES_FILENAME_PATTERN.format(command_name), + default=excludes_filename_default if os.path.exists(excludes_filename_default) else None, help='Excludes filename', ) parser.add_argument( diff --git a/atticmatic/tests/integration/test_command.py b/atticmatic/tests/integration/test_command.py index fd72595..346612b 100644 --- a/atticmatic/tests/integration/test_command.py +++ b/atticmatic/tests/integration/test_command.py @@ -1,5 +1,7 @@ +import os import sys +from flexmock import flexmock from nose.tools import assert_raises from atticmatic import command as module @@ -9,6 +11,8 @@ COMMAND_NAME = 'foomatic' def test_parse_arguments_with_no_arguments_uses_defaults(): + flexmock(os.path).should_receive('exists').and_return(True) + parser = module.parse_arguments(COMMAND_NAME) assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME) @@ -17,6 +21,8 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): def test_parse_arguments_with_filename_arguments_overrides_defaults(): + flexmock(os.path).should_receive('exists').and_return(True) + parser = module.parse_arguments(COMMAND_NAME, '--config', 'myconfig', '--excludes', 'myexcludes') assert parser.config_filename == 'myconfig' @@ -24,7 +30,29 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults(): assert parser.verbosity == None +def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none(): + flexmock(os.path).should_receive('exists').and_return(False) + + parser = module.parse_arguments(COMMAND_NAME) + + assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME) + assert parser.excludes_filename == None + assert parser.verbosity == None + + +def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename(): + flexmock(os.path).should_receive('exists').and_return(False) + + parser = module.parse_arguments(COMMAND_NAME, '--excludes', 'myexcludes') + + assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME) + assert parser.excludes_filename == 'myexcludes' + assert parser.verbosity == None + + def test_parse_arguments_with_verbosity_flag_overrides_default(): + flexmock(os.path).should_receive('exists').and_return(True) + parser = module.parse_arguments(COMMAND_NAME, '--verbosity', '1') assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME) @@ -33,6 +61,7 @@ def test_parse_arguments_with_verbosity_flag_overrides_default(): def test_parse_arguments_with_invalid_arguments_exits(): + flexmock(os.path).should_receive('exists').and_return(True) original_stderr = sys.stderr sys.stderr = sys.stdout diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py index c779c36..8fe236f 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -29,7 +29,8 @@ def insert_datetime_mock(): ).mock -CREATE_COMMAND = ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar') +CREATE_COMMAND_WITHOUT_EXCLUDES = ('attic', 'create', 'repo::host-now', 'foo', 'bar') +CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes') def test_create_archive_should_call_attic_with_parameters(): @@ -46,6 +47,20 @@ def test_create_archive_should_call_attic_with_parameters(): ) +def test_create_archive_with_none_excludes_filename_should_call_attic_without_excludes(): + insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + excludes_filename=None, + verbosity=None, + source_directories='foo bar', + repository='repo', + command='attic', + ) + + def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter(): insert_subprocess_mock(CREATE_COMMAND + ('--stats',)) insert_platform_mock() From c27b4a34976ba078857d61c47cb3d2f2befb9758 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 26 Jul 2015 21:06:06 -0700 Subject: [PATCH 061/189] Added tag 0.1.2 for changeset 83067f995dd3 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index cc49115..005ecdf 100644 --- a/.hgtags +++ b/.hgtags @@ -14,3 +14,4 @@ cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7 ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1 ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1 7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1 +83067f995dd391e38544a7722dc3b254b59c5521 0.1.2 From f94181480cefb6d8bf073d8b30b3c1538ef5795d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 26 Jul 2015 21:29:14 -0700 Subject: [PATCH 062/189] Removing some annoying Pelican metadata from docs. --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index b738c96..ce38d80 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ title: Atticmatic -date: -save_as: atticmatic/index.html ## Overview From 952a691f60b6d257be2d203dc51c9472754049f8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 26 Jul 2015 22:02:43 -0700 Subject: [PATCH 063/189] Linking to both Attic and Borg check docs from sample config. --- atticmatic/backends/borg.py | 1 - sample/config | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py index 96b5bff..5214ca1 100644 --- a/atticmatic/backends/borg.py +++ b/atticmatic/backends/borg.py @@ -10,4 +10,3 @@ COMMAND = 'borg' create_archive = partial(shared.create_archive, command=COMMAND) prune_archives = partial(shared.prune_archives, command=COMMAND) check_archives = partial(shared.check_archives, command=COMMAND) - diff --git a/sample/config b/sample/config index 4e09b04..7ee5256 100644 --- a/sample/config +++ b/sample/config @@ -20,5 +20,6 @@ keep_yearly: 1 [consistency] # Space-separated list of consistency checks to run: "repository", "archives", # or both. Defaults to both. Set to "disabled" to disable all consistency -# checks. See https://attic-backup.org/usage.html#attic-check for details. +# checks. See https://attic-backup.org/usage.html#attic-check or +# https://borgbackup.github.io/borgbackup/usage.html#borg-check for details. checks: repository archives From f5e0e10143a6dcc1a36a535b34b7420af819b0d4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Jul 2015 19:06:39 -0700 Subject: [PATCH 064/189] #6: Fixing example config file to use valid keep_within value. --- sample/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample/config b/sample/config index 7ee5256..c83c554 100644 --- a/sample/config +++ b/sample/config @@ -9,7 +9,7 @@ repository: user@backupserver:sourcehostname.attic # Retention policy for how many backups to keep in each category. See # https://attic-backup.org/usage.html#attic-prune or # https://borgbackup.github.io/borgbackup/usage.html#borg-prune for details. -#keep_within: 3h +#keep_within: 3H #keep_hourly: 24 keep_daily: 7 keep_weekly: 4 From 9c0687407311e1d01bc9ee08f7cc3f1a12d68719 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Jul 2015 21:47:52 -0700 Subject: [PATCH 065/189] #1: Add support for "borg check --last N" to Borg backend. --- NEWS | 4 ++ atticmatic/backends/attic.py | 2 +- atticmatic/backends/borg.py | 12 +++++ atticmatic/backends/shared.py | 42 ++++++++++++++-- atticmatic/command.py | 4 +- atticmatic/config.py | 48 ++++--------------- atticmatic/tests/unit/backends/test_shared.py | 26 +++++++--- atticmatic/tests/unit/test_config.py | 11 +++-- sample/config | 2 + 9 files changed, 94 insertions(+), 57 deletions(-) diff --git a/NEWS b/NEWS index db51ca3..207769e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.1.3 + + * #1: Add support for "borg check --last N" to Borg backend. + 0.1.2 * As a convenience to new users, allow a missing default excludes file. diff --git a/atticmatic/backends/attic.py b/atticmatic/backends/attic.py index a99c323..fcacce3 100644 --- a/atticmatic/backends/attic.py +++ b/atticmatic/backends/attic.py @@ -5,7 +5,7 @@ from atticmatic.backends import shared # An atticmatic backend that supports Attic for actually handling backups. COMMAND = 'attic' - +CONFIG_FORMAT = shared.CONFIG_FORMAT create_archive = partial(shared.create_archive, command=COMMAND) prune_archives = partial(shared.prune_archives, command=COMMAND) diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py index 5214ca1..bd6a386 100644 --- a/atticmatic/backends/borg.py +++ b/atticmatic/backends/borg.py @@ -1,10 +1,22 @@ from functools import partial +from atticmatic.config import Section_format, option from atticmatic.backends import shared # An atticmatic backend that supports Borg for actually handling backups. COMMAND = 'borg' +CONFIG_FORMAT = ( + shared.CONFIG_FORMAT[0], # location + shared.CONFIG_FORMAT[1], # retention + Section_format( + 'consistency', + ( + option('checks', required=False), + option('check_last', required=False), + ), + ) +) create_archive = partial(shared.create_archive, command=COMMAND) diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py index 8abd064..74f5533 100644 --- a/atticmatic/backends/shared.py +++ b/atticmatic/backends/shared.py @@ -3,6 +3,7 @@ import os import platform import subprocess +from atticmatic.config import Section_format, option from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS @@ -12,6 +13,34 @@ from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS # atticmatic.backends.borg. +CONFIG_FORMAT = ( + Section_format( + 'location', + ( + option('source_directories'), + option('repository'), + ), + ), + Section_format( + 'retention', + ( + option('keep_within', required=False), + option('keep_hourly', int, required=False), + option('keep_daily', int, required=False), + option('keep_weekly', int, required=False), + option('keep_monthly', int, required=False), + option('keep_yearly', int, required=False), + option('prefix', required=False), + ), + ), + Section_format( + 'consistency', + ( + option('checks', required=False), + ), + ) +) + def create_archive(excludes_filename, verbosity, source_directories, repository, command): ''' Given an excludes filename (or None), a vebosity flag, a space-separated list of source @@ -110,7 +139,7 @@ def _parse_checks(consistency_config): ) -def _make_check_flags(checks): +def _make_check_flags(checks, check_last=None): ''' Given a parsed sequence of checks, transform it into tuple of command-line flags. @@ -121,13 +150,17 @@ def _make_check_flags(checks): This will be returned as: ('--repository-only',) + + Additionally, if a check_last value is given, a "--last" flag will be added. Note that only + Borg supports this flag. ''' + last_flag = ('--last', check_last) if check_last else () if checks == DEFAULT_CHECKS: - return () + return last_flag return tuple( '--{}-only'.format(check) for check in checks - ) + ) + last_flag def check_archives(verbosity, repository, consistency_config, command): @@ -138,6 +171,7 @@ def check_archives(verbosity, repository, consistency_config, command): If there are no consistency checks to run, skip running them. ''' checks = _parse_checks(consistency_config) + check_last = consistency_config.get('check_last', None) if not checks: return @@ -149,7 +183,7 @@ def check_archives(verbosity, repository, consistency_config, command): full_command = ( command, 'check', repository, - ) + _make_check_flags(checks) + verbosity_flags + ) + _make_check_flags(checks, check_last) + verbosity_flags # The check command spews to stdout even without the verbose flag. Suppress it. stdout = None if verbosity_flags else open(os.devnull, 'w') diff --git a/atticmatic/command.py b/atticmatic/command.py index 13e7b43..0f512e1 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -60,9 +60,9 @@ def main(): try: command_name = os.path.basename(sys.argv[0]) args = parse_arguments(command_name, *sys.argv[1:]) - config = parse_configuration(args.config_filename) - repository = config.location['repository'] backend = load_backend(command_name) + config = parse_configuration(args.config_filename, backend.CONFIG_FORMAT) + repository = config.location['repository'] backend.create_archive(args.excludes_filename, args.verbosity, **config.location) backend.prune_archives(args.verbosity, repository, config.retention) diff --git a/atticmatic/config.py b/atticmatic/config.py index 7474f05..bc1a32d 100644 --- a/atticmatic/config.py +++ b/atticmatic/config.py @@ -20,35 +20,6 @@ def option(name, value_type=str, required=True): return Config_option(name, value_type, required) -CONFIG_FORMAT = ( - Section_format( - 'location', - ( - option('source_directories'), - option('repository'), - ), - ), - Section_format( - 'retention', - ( - option('keep_within', required=False), - option('keep_hourly', int, required=False), - option('keep_daily', int, required=False), - option('keep_weekly', int, required=False), - option('keep_monthly', int, required=False), - option('keep_yearly', int, required=False), - option('prefix', required=False), - ), - ), - Section_format( - 'consistency', - ( - option('checks', required=False), - ), - ) -) - - def validate_configuration_format(parser, config_format): ''' Given an open ConfigParser and an expected config file format, validate that the parsed @@ -110,11 +81,6 @@ def validate_configuration_format(parser, config_format): ) -# Describes a parsed configuration, where each attribute is the name of a configuration file section -# and each value is a dict of that section's parsed options. -Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT)) - - def parse_section_options(parser, section_format): ''' Given an open ConfigParser and an expected section format, return the option values from that @@ -135,21 +101,25 @@ def parse_section_options(parser, section_format): ) -def parse_configuration(config_filename): +def parse_configuration(config_filename, config_format): ''' - Given a config filename of the expected format, return the parsed configuration as Parsed_config - data structure. + Given a config filename and an expected config file format, return the parsed configuration + as a namedtuple with one attribute for each parsed section. Raise IOError if the file cannot be read, or ValueError if the format is not as expected. ''' parser = ConfigParser() parser.read(config_filename) - validate_configuration_format(parser, CONFIG_FORMAT) + validate_configuration_format(parser, config_format) + + # Describes a parsed configuration, where each attribute is the name of a configuration file + # section and each value is a dict of that section's parsed options. + Parsed_config = namedtuple('Parsed_config', (section_format.name for section_format in config_format)) return Parsed_config( *( parse_section_options(parser, section_format) - for section_format in CONFIG_FORMAT + for section_format in config_format ) ) diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py index 8fe236f..c742087 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -196,10 +196,24 @@ def test_make_check_flags_with_default_checks_returns_no_flags(): assert flags == () +def test_make_check_flags_with_checks_and_last_returns_flags_including_last(): + flags = module._make_check_flags(('foo', 'bar'), check_last=3) + + assert flags == ('--foo-only', '--bar-only', '--last', 3) + + +def test_make_check_flags_with_last_returns_last_flag(): + flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3) + + assert flags == ('--last', 3) + + def test_check_archives_should_call_attic_with_parameters(): - consistency_config = flexmock() - flexmock(module).should_receive('_parse_checks').and_return(flexmock()) - flexmock(module).should_receive('_make_check_flags').and_return(()) + checks = flexmock() + check_last = flexmock() + consistency_config = flexmock().should_receive('get').and_return(check_last).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) stdout = flexmock() insert_subprocess_mock( ('attic', 'check', 'repo'), @@ -219,7 +233,7 @@ def test_check_archives_should_call_attic_with_parameters(): def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter(): - consistency_config = flexmock() + consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( @@ -238,7 +252,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter(): - consistency_config = flexmock() + consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( @@ -257,7 +271,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param def test_check_archives_without_any_checks_should_bail(): - consistency_config = flexmock() + consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(()) insert_subprocess_never() diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py index 02f1f52..4c889b0 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/atticmatic/tests/unit/test_config.py @@ -205,17 +205,18 @@ def insert_mock_parser(): def test_parse_configuration_should_return_section_configs(): parser = insert_mock_parser() + config_format = (flexmock(name='items'), flexmock(name='things')) mock_module = flexmock(module) mock_module.should_receive('validate_configuration_format').with_args( - parser, module.CONFIG_FORMAT, + parser, config_format, ).once() - mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT) + mock_section_configs = (flexmock(), flexmock()) - for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs): + for section_format, section_config in zip(config_format, mock_section_configs): mock_module.should_receive('parse_section_options').with_args( parser, section_format, ).and_return(section_config).once() - parsed_config = module.parse_configuration('filename') + parsed_config = module.parse_configuration('filename', config_format) - assert parsed_config == module.Parsed_config(*mock_section_configs) + assert parsed_config == type(parsed_config)(*mock_section_configs) diff --git a/sample/config b/sample/config index c83c554..82d77d1 100644 --- a/sample/config +++ b/sample/config @@ -23,3 +23,5 @@ keep_yearly: 1 # checks. See https://attic-backup.org/usage.html#attic-check or # https://borgbackup.github.io/borgbackup/usage.html#borg-check for details. checks: repository archives +# For Borg only, you can restrict the number of checked archives to the last n. +#check_last: 3 From e996e09657becc1a936314254aa102143eeeb5d4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Jul 2015 21:48:21 -0700 Subject: [PATCH 066/189] Added tag 0.1.3 for changeset acc7fb61566f --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 005ecdf..e4e3ada 100644 --- a/.hgtags +++ b/.hgtags @@ -15,3 +15,4 @@ ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1 ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1 7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1 83067f995dd391e38544a7722dc3b254b59c5521 0.1.2 +acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3 From d041e23d35ddea84cff390bc80160af1284e8775 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 30 Jul 2015 08:12:31 -0700 Subject: [PATCH 067/189] Adding test that setup.py version matches release version. --- NEWS | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 207769e..367281e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.1.4 + + * Adding test that setup.py version matches release version. + 0.1.3 * #1: Add support for "borg check --last N" to Borg backend. diff --git a/setup.py b/setup.py index 7d535bc..5aee184 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='atticmatic', - version='0.1.1', + version='0.1.4', description='A wrapper script for Attic/Borg backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', From 61969d17a262f354d14083ec9e582474d336b67a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 30 Jul 2015 08:12:36 -0700 Subject: [PATCH 068/189] Added tag 0.1.4 for changeset 6dda59c12de8 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index e4e3ada..e8f26b9 100644 --- a/.hgtags +++ b/.hgtags @@ -16,3 +16,4 @@ ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1 7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1 83067f995dd391e38544a7722dc3b254b59c5521 0.1.2 acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3 +6dda59c12de88f060eb7244e6d330173985a9639 0.1.4 From 996ca19dac815b061e396a78e9a6cdda136e5246 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 30 Jul 2015 08:13:27 -0700 Subject: [PATCH 069/189] Adding version test. --- atticmatic/tests/integration/test_version.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 atticmatic/tests/integration/test_version.py diff --git a/atticmatic/tests/integration/test_version.py b/atticmatic/tests/integration/test_version.py new file mode 100644 index 0000000..e5ba486 --- /dev/null +++ b/atticmatic/tests/integration/test_version.py @@ -0,0 +1,8 @@ +import subprocess + + +def test_setup_version_matches_news_version(): + setup_version = subprocess.check_output(('python', 'setup.py', '--version')).decode('ascii') + news_version = open('NEWS').readline() + + assert setup_version == news_version From 1334da99e2b571ed897e104c658d490e7a7ef516 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 30 Jul 2015 08:13:32 -0700 Subject: [PATCH 070/189] Added tag 0.1.4 for changeset e58246fc92bb --- .hgtags | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.hgtags b/.hgtags index e8f26b9..844f466 100644 --- a/.hgtags +++ b/.hgtags @@ -17,3 +17,5 @@ ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1 83067f995dd391e38544a7722dc3b254b59c5521 0.1.2 acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3 6dda59c12de88f060eb7244e6d330173985a9639 0.1.4 +6dda59c12de88f060eb7244e6d330173985a9639 0.1.4 +e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4 From 204e515bf7b3ba16ff72e3dc26d3d58b6136eeb2 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Aug 2015 10:59:27 -0700 Subject: [PATCH 071/189] Changes to support release on PyPI. Now pip installable by name! --- .hgignore | 1 + NEWS | 4 ++++ README.md | 2 +- setup.cfg | 3 +++ setup.py | 17 ++++++++++++++++- 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.hgignore b/.hgignore index d7c266a..d814e38 100644 --- a/.hgignore +++ b/.hgignore @@ -3,3 +3,4 @@ syntax: glob *.pyc *.swp .tox +dist diff --git a/NEWS b/NEWS index 367281e..482da6b 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.1.5 + + * Changes to support release on PyPI. Now pip installable by name! + 0.1.4 * Adding test that setup.py version matches release version. diff --git a/README.md b/README.md index ce38d80..8ed8548 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ key-based ssh access to the desired user account on the remote host. To install atticmatic, run the following command to download and install it: - sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic + sudo pip install --upgrade atticmatic If you are using Attic, copy the following configuration files: diff --git a/setup.cfg b/setup.cfg index ead29d1..222a3da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ +[metadata] +description-file=README.md + [nosetests] detailed-errors=1 diff --git a/setup.py b/setup.py index 5aee184..ad32a14 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,26 @@ from setuptools import setup, find_packages + +VERSION = '0.1.5' + + setup( name='atticmatic', - version='0.1.4', + version=VERSION, description='A wrapper script for Attic/Borg backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', + url='https://torsion.org/atticmatic', + download_url='https://torsion.org/hg/atticmatic/archive/%s.tar.gz' % VERSION, + classifiers=( + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Programming Language :: Python', + 'Topic :: Security :: Cryptography', + 'Topic :: System :: Archiving :: Backup', + ), packages=find_packages(), entry_points={ 'console_scripts': [ From 5299046b6bb95134185836fa04e093ba5854b612 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Aug 2015 10:59:40 -0700 Subject: [PATCH 072/189] Added tag 0.1.5 for changeset 0afff209b902 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 844f466..076d808 100644 --- a/.hgtags +++ b/.hgtags @@ -19,3 +19,4 @@ acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3 6dda59c12de88f060eb7244e6d330173985a9639 0.1.4 6dda59c12de88f060eb7244e6d330173985a9639 0.1.4 e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4 +0afff209b902698c2266986129d6dc9f5f913101 0.1.5 From c67ab09e4d6f61b26605336bcecb870c62c7b555 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Aug 2015 11:04:57 -0700 Subject: [PATCH 073/189] Adding build to hgignore. --- .hgignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgignore b/.hgignore index d814e38..4128076 100644 --- a/.hgignore +++ b/.hgignore @@ -3,4 +3,5 @@ syntax: glob *.pyc *.swp .tox +build dist From 30f6ec4f7dee9d358c9dd62287c9b6c705977f8a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 2 Sep 2015 18:45:15 -0700 Subject: [PATCH 074/189] Adding documentation note about logging into the issue tracker in order to create issues. --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8ed8548..246b86a 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,8 @@ backups. ## Issues and feedback Got an issue or an idea for a feature enhancement? Check out the [atticmatic -issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues). +issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues). In +order to create a new issue or comment on an issue, you'll need to [login +first](https://tree.taiga.io/login). Other questions or comments? Contact . From 3a9e32a4119a0a26b2c4679e22f13af56700c4a6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 2 Sep 2015 22:48:07 -0700 Subject: [PATCH 075/189] #9: New configuration option for the encryption passphrase. #10: Support for Borg's new archive compression feature. --- NEWS | 5 +++ README.md | 6 +-- atticmatic/backends/attic.py | 2 + atticmatic/backends/borg.py | 10 ++++- atticmatic/backends/shared.py | 27 ++++++++++-- atticmatic/command.py | 5 ++- atticmatic/tests/unit/backends/test_shared.py | 42 +++++++++++++++++++ sample/config | 9 ++++ setup.py | 2 +- 9 files changed, 98 insertions(+), 10 deletions(-) diff --git a/NEWS b/NEWS index 482da6b..d518502 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +0.1.6 + + * #9: New configuration option for the encryption passphrase. + * #10: Support for Borg's new archive compression feature. + 0.1.5 * Changes to support release on PyPI. Now pip installable by name! diff --git a/README.md b/README.md index 246b86a..3d9d999 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,9 @@ Start](https://attic-backup.org/quickstart.html) or the [Borg Quick Start](https://borgbackup.github.io/borgbackup/quickstart.html) to create a repository on a local or remote host. Note that if you plan to run atticmatic on a schedule with cron, and you encrypt your attic repository with a -passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE` -environment variable. See the repository encryption section of the Quick Start -for more info. +passphrase instead of a key file, you'll need to set the atticmatic +`encryption_passphrase` configuration variable. See the repository encryption +section of the Quick Start for more info. If the repository is on a remote host, make sure that your local root user has key-based ssh access to the desired user account on the remote host. diff --git a/atticmatic/backends/attic.py b/atticmatic/backends/attic.py index fcacce3..daa0b29 100644 --- a/atticmatic/backends/attic.py +++ b/atticmatic/backends/attic.py @@ -7,6 +7,8 @@ from atticmatic.backends import shared COMMAND = 'attic' CONFIG_FORMAT = shared.CONFIG_FORMAT + +initialize = partial(shared.initialize, command=COMMAND) create_archive = partial(shared.create_archive, command=COMMAND) prune_archives = partial(shared.prune_archives, command=COMMAND) check_archives = partial(shared.check_archives, command=COMMAND) diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py index bd6a386..222c702 100644 --- a/atticmatic/backends/borg.py +++ b/atticmatic/backends/borg.py @@ -8,7 +8,14 @@ from atticmatic.backends import shared COMMAND = 'borg' CONFIG_FORMAT = ( shared.CONFIG_FORMAT[0], # location - shared.CONFIG_FORMAT[1], # retention + Section_format( + 'storage', + ( + option('encryption_passphrase', required=False), + option('compression', required=False), + ), + ), + shared.CONFIG_FORMAT[2], # retention Section_format( 'consistency', ( @@ -19,6 +26,7 @@ CONFIG_FORMAT = ( ) +initialize = partial(shared.initialize, command=COMMAND) create_archive = partial(shared.create_archive, command=COMMAND) prune_archives = partial(shared.prune_archives, command=COMMAND) check_archives = partial(shared.check_archives, command=COMMAND) diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py index 74f5533..5baff3a 100644 --- a/atticmatic/backends/shared.py +++ b/atticmatic/backends/shared.py @@ -21,6 +21,12 @@ CONFIG_FORMAT = ( option('repository'), ), ), + Section_format( + 'storage', + ( + option('encryption_passphrase', required=False), + ), + ), Section_format( 'retention', ( @@ -41,13 +47,26 @@ CONFIG_FORMAT = ( ) ) -def create_archive(excludes_filename, verbosity, source_directories, repository, command): + +def initialize(storage_config, command): + passphrase = storage_config.get('encryption_passphrase') + + if passphrase: + os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase + + +def create_archive( + excludes_filename, verbosity, storage_config, source_directories, repository, command, +): ''' - Given an excludes filename (or None), a vebosity flag, a space-separated list of source - directories, a local or remote repository path, and a command to run, create an attic archive. + Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated + list of source directories, a local or remote repository path, and a command to run, create an + attic archive. ''' sources = tuple(source_directories.split(' ')) exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else () + compression = storage_config.get('compression', None) + compression_flags = ('--compression', compression) if compression else () verbosity_flags = { VERBOSITY_SOME: ('--stats',), VERBOSITY_LOTS: ('--verbose', '--stats'), @@ -60,7 +79,7 @@ def create_archive(excludes_filename, verbosity, source_directories, repository, hostname=platform.node(), timestamp=datetime.now().isoformat(), ), - ) + sources + exclude_flags + verbosity_flags + ) + sources + exclude_flags + compression_flags + verbosity_flags subprocess.check_call(full_command) diff --git a/atticmatic/command.py b/atticmatic/command.py index 0f512e1..08ea49e 100644 --- a/atticmatic/command.py +++ b/atticmatic/command.py @@ -64,7 +64,10 @@ def main(): config = parse_configuration(args.config_filename, backend.CONFIG_FORMAT) repository = config.location['repository'] - backend.create_archive(args.excludes_filename, args.verbosity, **config.location) + backend.initialize(config.storage) + backend.create_archive( + args.excludes_filename, args.verbosity, config.storage, **config.location + ) backend.prune_archives(args.verbosity, repository, config.retention) backend.check_archives(args.verbosity, repository, config.consistency) except (ValueError, IOError, CalledProcessError) as error: diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py index c742087..6449231 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -1,4 +1,5 @@ from collections import OrderedDict +import os from flexmock import flexmock @@ -7,6 +8,28 @@ from atticmatic.tests.builtins import builtins_mock from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS +def test_initialize_with_passphrase_should_set_environment(): + orig_environ = os.environ + + try: + os.environ = {} + module.initialize({'encryption_passphrase': 'pass'}, command='attic') + assert os.environ.get('ATTIC_PASSPHRASE') == 'pass' + finally: + os.environ = orig_environ + + +def test_initialize_without_passphrase_should_not_set_environment(): + orig_environ = os.environ + + try: + os.environ = {} + module.initialize({}, command='attic') + assert os.environ.get('ATTIC_PASSPHRASE') == None + finally: + os.environ = orig_environ + + def insert_subprocess_mock(check_call_command, **kwargs): subprocess = flexmock() subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() @@ -41,6 +64,7 @@ def test_create_archive_should_call_attic_with_parameters(): module.create_archive( excludes_filename='excludes', verbosity=None, + storage_config={}, source_directories='foo bar', repository='repo', command='attic', @@ -55,6 +79,7 @@ def test_create_archive_with_none_excludes_filename_should_call_attic_without_ex module.create_archive( excludes_filename=None, verbosity=None, + storage_config={}, source_directories='foo bar', repository='repo', command='attic', @@ -69,6 +94,7 @@ def test_create_archive_with_verbosity_some_should_call_attic_with_stats_paramet module.create_archive( excludes_filename='excludes', verbosity=VERBOSITY_SOME, + storage_config={}, source_directories='foo bar', repository='repo', command='attic', @@ -83,6 +109,22 @@ def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_param module.create_archive( excludes_filename='excludes', verbosity=VERBOSITY_LOTS, + storage_config={}, + source_directories='foo bar', + repository='repo', + command='attic', + ) + + +def test_create_archive_with_compression_should_call_attic_with_compression_parameters(): + insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + excludes_filename='excludes', + verbosity=None, + storage_config={'compression': 'rle'}, source_directories='foo bar', repository='repo', command='attic', diff --git a/sample/config b/sample/config index 82d77d1..cf4b391 100644 --- a/sample/config +++ b/sample/config @@ -5,6 +5,15 @@ source_directories: /home /etc # Path to local or remote repository. repository: user@backupserver:sourcehostname.attic +[storage] +# Passphrase to unlock the encryption key with. Only use on repositories that +# were initialized with passphrase/repokey encryption. +#encryption_passphrase: foo +# For Borg only, you can specify the type of compression to use when creating +# archives. See https://borgbackup.github.io/borgbackup/usage.html#borg-create +# for details. Defaults to no compression. +#compression: lz4 + [retention] # Retention policy for how many backups to keep in each category. See # https://attic-backup.org/usage.html#attic-prune or diff --git a/setup.py b/setup.py index ad32a14..73a7796 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '0.1.5' +VERSION = '0.1.6' setup( From 5c58f85be18be79de0785c025b517dbc88a85f17 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 2 Sep 2015 22:48:14 -0700 Subject: [PATCH 076/189] Added tag 0.1.6 for changeset 4c63f3d90ec2 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 076d808..fed1728 100644 --- a/.hgtags +++ b/.hgtags @@ -20,3 +20,4 @@ acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3 6dda59c12de88f060eb7244e6d330173985a9639 0.1.4 e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4 0afff209b902698c2266986129d6dc9f5f913101 0.1.5 +4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6 From 6dc0173b74da512896e40e22009f71fd51cb0df7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 6 Sep 2015 15:33:56 -0700 Subject: [PATCH 077/189] #11: Fixed parsing of punctuation in configuration file. --- NEWS | 4 +++ atticmatic/config.py | 10 +++---- atticmatic/tests/integration/test_config.py | 29 +++++++++++++++++++++ atticmatic/tests/unit/test_config.py | 2 +- setup.py | 2 +- 5 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 atticmatic/tests/integration/test_config.py diff --git a/NEWS b/NEWS index d518502..2a7c7ae 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.1.7-dev + + * #11: Fixed parsing of punctuation in configuration file. + 0.1.6 * #9: New configuration option for the encryption passphrase. diff --git a/atticmatic/config.py b/atticmatic/config.py index bc1a32d..4f45b89 100644 --- a/atticmatic/config.py +++ b/atticmatic/config.py @@ -2,10 +2,10 @@ from collections import OrderedDict, namedtuple try: # Python 2 - from ConfigParser import ConfigParser + from ConfigParser import RawConfigParser except ImportError: # Python 3 - from configparser import ConfigParser + from configparser import RawConfigParser Section_format = namedtuple('Section_format', ('name', 'options')) @@ -22,7 +22,7 @@ def option(name, value_type=str, required=True): def validate_configuration_format(parser, config_format): ''' - Given an open ConfigParser and an expected config file format, validate that the parsed + Given an open RawConfigParser and an expected config file format, validate that the parsed configuration file has the expected sections, that any required options are present in those sections, and that there aren't any unexpected options. @@ -83,7 +83,7 @@ def validate_configuration_format(parser, config_format): def parse_section_options(parser, section_format): ''' - Given an open ConfigParser and an expected section format, return the option values from that + Given an open RawConfigParser and an expected section format, return the option values from that section as a dict mapping from option name to value. Omit those options that are not present in the parsed options. @@ -108,7 +108,7 @@ def parse_configuration(config_filename, config_format): Raise IOError if the file cannot be read, or ValueError if the format is not as expected. ''' - parser = ConfigParser() + parser = RawConfigParser() parser.read(config_filename) validate_configuration_format(parser, config_format) diff --git a/atticmatic/tests/integration/test_config.py b/atticmatic/tests/integration/test_config.py new file mode 100644 index 0000000..31fcd9e --- /dev/null +++ b/atticmatic/tests/integration/test_config.py @@ -0,0 +1,29 @@ +try: + # Python 2 + from cStringIO import StringIO +except ImportError: + # Python 3 + from io import StringIO + +from collections import OrderedDict +import string + +from atticmatic import config as module + + +def test_parse_section_options_with_punctuation_should_return_section_options(): + parser = module.RawConfigParser() + parser.readfp(StringIO('[section]\nfoo: {}\n'.format(string.punctuation))) + + section_format = module.Section_format( + 'section', + (module.Config_option('foo', str, required=True),), + ) + + config = module.parse_section_options(parser, section_format) + + assert config == OrderedDict( + ( + ('foo', string.punctuation), + ) + ) diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py index 4c889b0..7a1866e 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/atticmatic/tests/unit/test_config.py @@ -198,7 +198,7 @@ def test_parse_section_options_for_missing_section_should_return_empty_dict(): def insert_mock_parser(): parser = flexmock() parser.should_receive('read') - module.ConfigParser = lambda: parser + module.RawConfigParser = lambda: parser return parser diff --git a/setup.py b/setup.py index 73a7796..9dc31c7 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '0.1.6' +VERSION = '0.1.7-dev' setup( From 8a58b72934c4f84d667dcce0402e0789f93c3af1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 6 Sep 2015 15:55:14 -0700 Subject: [PATCH 078/189] Better error message when configuration file is missing. --- NEWS | 1 + atticmatic/config.py | 3 ++- atticmatic/tests/unit/test_config.py | 10 +++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 2a7c7ae..1b46c71 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 0.1.7-dev * #11: Fixed parsing of punctuation in configuration file. + * Better error message when configuration file is missing. 0.1.6 diff --git a/atticmatic/config.py b/atticmatic/config.py index 4f45b89..ddf574d 100644 --- a/atticmatic/config.py +++ b/atticmatic/config.py @@ -109,7 +109,8 @@ def parse_configuration(config_filename, config_format): Raise IOError if the file cannot be read, or ValueError if the format is not as expected. ''' parser = RawConfigParser() - parser.read(config_filename) + if not parser.read(config_filename): + raise ValueError('Configuration file cannot be opened: {}'.format(config_filename)) validate_configuration_format(parser, config_format) diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py index 7a1866e..4b4d3a7 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/atticmatic/tests/unit/test_config.py @@ -197,7 +197,7 @@ def test_parse_section_options_for_missing_section_should_return_empty_dict(): def insert_mock_parser(): parser = flexmock() - parser.should_receive('read') + parser.should_receive('read').and_return([flexmock()]) module.RawConfigParser = lambda: parser return parser @@ -220,3 +220,11 @@ def test_parse_configuration_should_return_section_configs(): parsed_config = module.parse_configuration('filename', config_format) assert parsed_config == type(parsed_config)(*mock_section_configs) + + +def test_parse_configuration_with_file_open_error_should_raise(): + parser = insert_mock_parser() + parser.should_receive('read').and_return([]) + + with assert_raises(ValueError): + module.parse_configuration('filename', config_format=flexmock()) From 2456fc67f17679204b9a1052c80d5b0cc462f54d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 6 Sep 2015 16:40:39 -0700 Subject: [PATCH 079/189] Revving version. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 1b46c71..387a46d 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -0.1.7-dev +0.1.7 * #11: Fixed parsing of punctuation in configuration file. * Better error message when configuration file is missing. diff --git a/setup.py b/setup.py index 9dc31c7..343d69f 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '0.1.7-dev' +VERSION = '0.1.7' setup( From 944c0212c38690e3120541b3b0a244d23ed1743e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 6 Sep 2015 16:40:46 -0700 Subject: [PATCH 080/189] Added tag 0.1.7 for changeset 5a458ebef804 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index fed1728..072fbd2 100644 --- a/.hgtags +++ b/.hgtags @@ -21,3 +21,4 @@ acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3 e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4 0afff209b902698c2266986129d6dc9f5f913101 0.1.5 4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6 +5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7 From fa7955b8cf9a989ca35cbd3b34b50dfc6a4eb771 Mon Sep 17 00:00:00 2001 From: TW Date: Tue, 20 Oct 2015 23:08:43 +0200 Subject: [PATCH 081/189] fixed typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3d9d999..9c38cb5 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ Then, to actually run tests, run: ### Broken pipe with remote repository When running atticmatic on a large remote repository, you may receive errors -like the following, particularly while "attic check" is valiating backups for +like the following, particularly while "attic check" is validating backups for consistency: Write failed: Broken pipe From 80318e6e30fd90d1dce38e862159cd983450dd3d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 8 Nov 2015 17:03:40 -0800 Subject: [PATCH 082/189] Removed tag github/yaml_config_files --- .hgtags | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.hgtags b/.hgtags index 072fbd2..0e2b569 100644 --- a/.hgtags +++ b/.hgtags @@ -22,3 +22,5 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4 0afff209b902698c2266986129d6dc9f5f913101 0.1.5 4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6 5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7 +977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files +0000000000000000000000000000000000000000 github/yaml_config_files From 3a3851d2a5d6309a4672b7e2d26f91403bdda1eb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 8 Nov 2015 17:04:14 -0800 Subject: [PATCH 083/189] Removed tag github/yaml_config_files --- .hgtags | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.hgtags b/.hgtags index 072fbd2..0e2b569 100644 --- a/.hgtags +++ b/.hgtags @@ -22,3 +22,5 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4 0afff209b902698c2266986129d6dc9f5f913101 0.1.5 4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6 5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7 +977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files +0000000000000000000000000000000000000000 github/yaml_config_files From e59845d4e18b339e49f66cd8c0b9b3828d42e8f7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 8 Nov 2015 17:06:48 -0800 Subject: [PATCH 084/189] Added tag github/master for changeset 28434dd0440c --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 0e2b569..b580959 100644 --- a/.hgtags +++ b/.hgtags @@ -24,3 +24,4 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4 5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7 977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files 0000000000000000000000000000000000000000 github/yaml_config_files +28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master From 32858fb0b4b97fc73f7969439c5207b90c7f97ca Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Wed, 20 Jan 2016 13:11:15 +0100 Subject: [PATCH 085/189] Also allow the INI example to be highlighted on GitHub. --- README.md | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9c38cb5..509b1dd 100644 --- a/README.md +++ b/README.md @@ -12,22 +12,24 @@ all on the command-line, and handles common errors. Here's an example config file: - [location] - # Space-separated list of source directories to backup. - source_directories: /home /etc +```INI +[location] +# Space-separated list of source directories to backup. +source_directories: /home /etc - # Path to local or remote backup repository. - repository: user@backupserver:sourcehostname.attic +# Path to local or remote backup repository. +repository: user@backupserver:sourcehostname.attic - [retention] - # Retention policy for how many backups to keep in each category. - keep_daily: 7 - keep_weekly: 4 - keep_monthly: 6 +[retention] +# Retention policy for how many backups to keep in each category. +keep_daily: 7 +keep_weekly: 4 +keep_monthly: 6 - [consistency] - # Consistency checks to run, or "disabled" to prevent checks. - checks: repository archives +[consistency] +# Consistency checks to run, or "disabled" to prevent checks. +checks: repository archives +``` Additionally, exclude patterns can be specified in a separate excludes config file, one pattern per line. From 978096b402de91cdb1c1064a84ca0596504d040d Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sat, 30 Jan 2016 20:24:29 +0100 Subject: [PATCH 086/189] Added .gitignore file. --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a53007 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.egg-info +*.pyc +*.swp +.tox +build +dist From 9e52be6ffd6d7d1b79b5bb179859ccceba89c7af Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sun, 31 Jan 2016 11:42:07 +0100 Subject: [PATCH 087/189] Use /\s+/ to split source_directories to handle 1+ spaces. This bug is can be quite annoying because when you accidentally used something like: ```ini [location] source_directories: backup_one backup_two ; A (Additional space here) ``` It would call Attic/Borg with ('backup_one', '', 'backup_two') which in turn backups your whole $PWD. --- NEWS | 4 ++++ atticmatic/backends/shared.py | 5 +++-- atticmatic/tests/unit/backends/test_shared.py | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 387a46d..b7227b8 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +0.1.8-dev + + * Fixed handling of repeated spaces in source_directories which resulted in backup up everything. + 0.1.7 * #11: Fixed parsing of punctuation in configuration file. diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py index 5baff3a..a60aa3b 100644 --- a/atticmatic/backends/shared.py +++ b/atticmatic/backends/shared.py @@ -1,5 +1,6 @@ from datetime import datetime import os +import re import platform import subprocess @@ -63,7 +64,7 @@ def create_archive( list of source directories, a local or remote repository path, and a command to run, create an attic archive. ''' - sources = tuple(source_directories.split(' ')) + sources = tuple(re.split('\s+', source_directories)) exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else () compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () @@ -167,7 +168,7 @@ def _make_check_flags(checks, check_last=None): ('repository',) This will be returned as: - + ('--repository-only',) Additionally, if a check_last value is given, a "--last" flag will be added. Note that only diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py index 6449231..ddb090f 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -71,6 +71,21 @@ def test_create_archive_should_call_attic_with_parameters(): ) +def test_create_archive_with_two_spaces_in_source_directories(): + insert_subprocess_mock(CREATE_COMMAND) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + excludes_filename='excludes', + verbosity=None, + storage_config={}, + source_directories='foo bar', + repository='repo', + command='attic', + ) + + def test_create_archive_with_none_excludes_filename_should_call_attic_without_excludes(): insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES) insert_platform_mock() From 049f9c8853ef287a547e55ab2b3d02ea6b0c6cd7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 13 Feb 2016 10:43:31 -0800 Subject: [PATCH 088/189] Added support for --one-file-system for Borg. --- AUTHORS | 1 + NEWS | 3 ++- atticmatic/backends/borg.py | 9 ++++++++- atticmatic/backends/shared.py | 5 ++++- atticmatic/config.py | 1 + atticmatic/tests/unit/backends/test_shared.py | 16 ++++++++++++++++ atticmatic/tests/unit/test_config.py | 2 ++ sample/config | 4 ++++ setup.py | 2 +- 9 files changed, 39 insertions(+), 4 deletions(-) diff --git a/AUTHORS b/AUTHORS index 2a2f28f..95d2df8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,3 +2,4 @@ Dan Helfman : Main developer Alexander Görtz: Python 3 compatibility Henning Schroeder: Copy editing +Robin `ypid` Schneider: Support additional options of Borg diff --git a/NEWS b/NEWS index b7227b8..1b0963f 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ -0.1.8-dev +0.1.8.dev0 * Fixed handling of repeated spaces in source_directories which resulted in backup up everything. + * Added support for --one-file-system for Borg. 0.1.7 diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py index 222c702..6105a05 100644 --- a/atticmatic/backends/borg.py +++ b/atticmatic/backends/borg.py @@ -7,7 +7,14 @@ from atticmatic.backends import shared COMMAND = 'borg' CONFIG_FORMAT = ( - shared.CONFIG_FORMAT[0], # location + Section_format( + 'location', + ( + option('source_directories'), + option('one_file_system', value_type=bool, required=False), + option('repository'), + ), + ), Section_format( 'storage', ( diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py index a60aa3b..5544b30 100644 --- a/atticmatic/backends/shared.py +++ b/atticmatic/backends/shared.py @@ -58,6 +58,7 @@ def initialize(storage_config, command): def create_archive( excludes_filename, verbosity, storage_config, source_directories, repository, command, + one_file_system=None, ): ''' Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated @@ -68,6 +69,7 @@ def create_archive( exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else () compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () + one_file_system_flags = ('--one-file-system',) if one_file_system else () verbosity_flags = { VERBOSITY_SOME: ('--stats',), VERBOSITY_LOTS: ('--verbose', '--stats'), @@ -80,7 +82,8 @@ def create_archive( hostname=platform.node(), timestamp=datetime.now().isoformat(), ), - ) + sources + exclude_flags + compression_flags + verbosity_flags + ) + sources + exclude_flags + compression_flags + one_file_system_flags + \ + verbosity_flags subprocess.check_call(full_command) diff --git a/atticmatic/config.py b/atticmatic/config.py index ddf574d..8f7ae9a 100644 --- a/atticmatic/config.py +++ b/atticmatic/config.py @@ -92,6 +92,7 @@ def parse_section_options(parser, section_format): type_getter = { str: parser.get, int: parser.getint, + bool: parser.getboolean, } return OrderedDict( diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py index ddb090f..660fc5b 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -146,6 +146,22 @@ def test_create_archive_with_compression_should_call_attic_with_compression_para ) +def test_create_archive_with_one_file_system_should_call_attic_with_one_file_system_parameters(): + insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + excludes_filename='excludes', + verbosity=None, + storage_config={}, + source_directories='foo bar', + repository='repo', + command='attic', + one_file_system=True, + ) + + BASE_PRUNE_FLAGS = ( ('--keep-daily', '1'), ('--keep-weekly', '2'), diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py index 4b4d3a7..88569e1 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/atticmatic/tests/unit/test_config.py @@ -154,6 +154,7 @@ def test_parse_section_options_should_return_section_options(): parser = flexmock() parser.should_receive('get').with_args('section', 'foo').and_return('value') parser.should_receive('getint').with_args('section', 'bar').and_return(1) + parser.should_receive('getboolean').never() parser.should_receive('has_option').with_args('section', 'foo').and_return(True) parser.should_receive('has_option').with_args('section', 'bar').and_return(True) @@ -179,6 +180,7 @@ def test_parse_section_options_for_missing_section_should_return_empty_dict(): parser = flexmock() parser.should_receive('get').never() parser.should_receive('getint').never() + parser.should_receive('getboolean').never() parser.should_receive('has_option').with_args('section', 'foo').and_return(False) parser.should_receive('has_option').with_args('section', 'bar').and_return(False) diff --git a/sample/config b/sample/config index cf4b391..3d9432f 100644 --- a/sample/config +++ b/sample/config @@ -2,6 +2,10 @@ # Space-separated list of source directories to backup. source_directories: /home /etc +# For Borg only, you can specify to stay in same file system (do not cross +# mount points). +#one_file_system: True + # Path to local or remote repository. repository: user@backupserver:sourcehostname.attic diff --git a/setup.py b/setup.py index 343d69f..ff5a4be 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '0.1.7' +VERSION = '0.1.8.dev0' setup( From 0012e0cdea8d4b6b57c50721ad39e946d95559b5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 13 Feb 2016 10:59:43 -0800 Subject: [PATCH 089/189] Support borg create --umask. (Merge PR from ypid.) --- NEWS | 1 + atticmatic/backends/borg.py | 2 ++ atticmatic/backends/shared.py | 4 +++- atticmatic/tests/unit/backends/test_shared.py | 15 +++++++++++++++ sample/config | 2 ++ 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 1b0963f..412e8bc 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ * Fixed handling of repeated spaces in source_directories which resulted in backup up everything. * Added support for --one-file-system for Borg. + * Support borg create --umask. 0.1.7 diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py index 6105a05..d99af4a 100644 --- a/atticmatic/backends/borg.py +++ b/atticmatic/backends/borg.py @@ -20,6 +20,7 @@ CONFIG_FORMAT = ( ( option('encryption_passphrase', required=False), option('compression', required=False), + option('umask', required=False), ), ), shared.CONFIG_FORMAT[2], # retention @@ -34,6 +35,7 @@ CONFIG_FORMAT = ( initialize = partial(shared.initialize, command=COMMAND) + create_archive = partial(shared.create_archive, command=COMMAND) prune_archives = partial(shared.prune_archives, command=COMMAND) check_archives = partial(shared.check_archives, command=COMMAND) diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py index 5544b30..f300d1e 100644 --- a/atticmatic/backends/shared.py +++ b/atticmatic/backends/shared.py @@ -69,6 +69,8 @@ def create_archive( exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else () compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () + umask = storage_config.get('umask', None) + umask_flags = ('--umask', str(umask)) if umask else () one_file_system_flags = ('--one-file-system',) if one_file_system else () verbosity_flags = { VERBOSITY_SOME: ('--stats',), @@ -83,7 +85,7 @@ def create_archive( timestamp=datetime.now().isoformat(), ), ) + sources + exclude_flags + compression_flags + one_file_system_flags + \ - verbosity_flags + umask_flags + verbosity_flags subprocess.check_call(full_command) diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py index 660fc5b..8babc58 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -162,6 +162,21 @@ def test_create_archive_with_one_file_system_should_call_attic_with_one_file_sys ) +def test_create_archive_with_umask_should_call_attic_with_umask_parameters(): + insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + excludes_filename='excludes', + verbosity=None, + storage_config={'umask': 740}, + source_directories='foo bar', + repository='repo', + command='attic', + ) + + BASE_PRUNE_FLAGS = ( ('--keep-daily', '1'), ('--keep-weekly', '2'), diff --git a/sample/config b/sample/config index 3d9432f..70527e4 100644 --- a/sample/config +++ b/sample/config @@ -17,6 +17,8 @@ repository: user@backupserver:sourcehostname.attic # archives. See https://borgbackup.github.io/borgbackup/usage.html#borg-create # for details. Defaults to no compression. #compression: lz4 +# For Borg only, you can specify the umask to be used for borg create. +#umask: 0740 [retention] # Retention policy for how many backups to keep in each category. See From 88da0c30392b8faa62d7c46d6265a735e32fbbc0 Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Mon, 25 Jan 2016 23:52:16 +0100 Subject: [PATCH 090/189] Added support for file globs in source_directories. source_directories_glob can be used to enable glob support (defaults to disabled). --- README.md | 1 + atticmatic/backends/shared.py | 7 ++++++- atticmatic/tests/unit/backends/test_shared.py | 16 ++++++++++++++++ sample/config | 1 + 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 509b1dd..d29687e 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Here's an example config file: [location] # Space-separated list of source directories to backup. source_directories: /home /etc +# source_directories_glob: 1 # Path to local or remote backup repository. repository: user@backupserver:sourcehostname.attic diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py index f300d1e..65a39df 100644 --- a/atticmatic/backends/shared.py +++ b/atticmatic/backends/shared.py @@ -3,6 +3,8 @@ import os import re import platform import subprocess +from glob import glob +from itertools import chain from atticmatic.config import Section_format, option from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS @@ -19,6 +21,7 @@ CONFIG_FORMAT = ( 'location', ( option('source_directories'), + option('source_directories_glob', int, required=False), option('repository'), ), ), @@ -58,7 +61,7 @@ def initialize(storage_config, command): def create_archive( excludes_filename, verbosity, storage_config, source_directories, repository, command, - one_file_system=None, + one_file_system=None, source_directories_glob=None ): ''' Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated @@ -66,6 +69,8 @@ def create_archive( attic archive. ''' sources = tuple(re.split('\s+', source_directories)) + if source_directories_glob: + sources = tuple(chain.from_iterable([glob(x) for x in sources])) exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else () compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py index 8babc58..684c6d6 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -177,6 +177,22 @@ def test_create_archive_with_umask_should_call_attic_with_umask_parameters(): ) +def test_create_archive_with_globs(): + insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'setup.py', 'setup.cfg')) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + excludes_filename=None, + verbosity=None, + storage_config={}, + source_directories='setup*', + repository='repo', + command='attic', + source_directories_glob=1, + ) + + BASE_PRUNE_FLAGS = ( ('--keep-daily', '1'), ('--keep-weekly', '2'), diff --git a/sample/config b/sample/config index 70527e4..c698a91 100644 --- a/sample/config +++ b/sample/config @@ -1,6 +1,7 @@ [location] # Space-separated list of source directories to backup. source_directories: /home /etc +# source_directories_glob: 1 # For Borg only, you can specify to stay in same file system (do not cross # mount points). From 953d08ba63ee93f0079a92ef1f50f6dd363d423e Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sat, 13 Feb 2016 21:05:34 +0100 Subject: [PATCH 091/189] Made globing for source_directories the default. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don’t remove non existing files/directories from the list and let attic/borg handle this. --- NEWS | 1 + README.md | 4 ++-- atticmatic/backends/shared.py | 8 +++----- atticmatic/tests/unit/backends/test_shared.py | 1 - sample/config | 4 ++-- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/NEWS b/NEWS index 412e8bc..77042d9 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ * Fixed handling of repeated spaces in source_directories which resulted in backup up everything. * Added support for --one-file-system for Borg. * Support borg create --umask. + * Added support for file globs in source_directories. 0.1.7 diff --git a/README.md b/README.md index d29687e..1082ff8 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ Here's an example config file: ```INI [location] # Space-separated list of source directories to backup. -source_directories: /home /etc -# source_directories_glob: 1 +# Globs are expanded. +source_directories: /home /etc /var/log/syslog* # Path to local or remote backup repository. repository: user@backupserver:sourcehostname.attic diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py index 65a39df..8026540 100644 --- a/atticmatic/backends/shared.py +++ b/atticmatic/backends/shared.py @@ -21,7 +21,6 @@ CONFIG_FORMAT = ( 'location', ( option('source_directories'), - option('source_directories_glob', int, required=False), option('repository'), ), ), @@ -61,16 +60,15 @@ def initialize(storage_config, command): def create_archive( excludes_filename, verbosity, storage_config, source_directories, repository, command, - one_file_system=None, source_directories_glob=None + one_file_system=None ): ''' Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated list of source directories, a local or remote repository path, and a command to run, create an attic archive. ''' - sources = tuple(re.split('\s+', source_directories)) - if source_directories_glob: - sources = tuple(chain.from_iterable([glob(x) for x in sources])) + sources = re.split('\s+', source_directories) + sources = tuple(chain.from_iterable([glob(x) if glob(x) else [x] for x in sources])) exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else () compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py index 684c6d6..20a98f6 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -189,7 +189,6 @@ def test_create_archive_with_globs(): source_directories='setup*', repository='repo', command='attic', - source_directories_glob=1, ) diff --git a/sample/config b/sample/config index c698a91..e681376 100644 --- a/sample/config +++ b/sample/config @@ -1,7 +1,7 @@ [location] # Space-separated list of source directories to backup. -source_directories: /home /etc -# source_directories_glob: 1 +# Globs are expanded. +source_directories: /home /etc /var/log/syslog* # For Borg only, you can specify to stay in same file system (do not cross # mount points). From cf545ae93a26bfb2ed023c2407f09d6601ee761a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 13 Feb 2016 16:41:17 -0800 Subject: [PATCH 092/189] Mocking out glob() in test so it doesn't hit the filesystem, and simplifying comprehension. --- atticmatic/backends/shared.py | 2 +- atticmatic/tests/unit/backends/test_shared.py | 39 +++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py index 8026540..417a732 100644 --- a/atticmatic/backends/shared.py +++ b/atticmatic/backends/shared.py @@ -68,7 +68,7 @@ def create_archive( attic archive. ''' sources = re.split('\s+', source_directories) - sources = tuple(chain.from_iterable([glob(x) if glob(x) else [x] for x in sources])) + sources = tuple(chain.from_iterable(glob(x) or [x] for x in sources)) exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else () compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py index 20a98f6..cddef00 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -177,16 +177,49 @@ def test_create_archive_with_umask_should_call_attic_with_umask_parameters(): ) -def test_create_archive_with_globs(): - insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'setup.py', 'setup.cfg')) +def test_create_archive_with_source_directories_glob_expands(): + insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() + flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( excludes_filename=None, verbosity=None, storage_config={}, - source_directories='setup*', + source_directories='foo*', + repository='repo', + command='attic', + ) + + +def test_create_archive_with_non_matching_source_directories_glob_passes_through(): + insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo*')) + insert_platform_mock() + insert_datetime_mock() + flexmock(module).should_receive('glob').with_args('foo*').and_return([]) + + module.create_archive( + excludes_filename=None, + verbosity=None, + storage_config={}, + source_directories='foo*', + repository='repo', + command='attic', + ) + + +def test_create_archive_with_glob_should_call_attic_with_expanded_directories(): + insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food')) + insert_platform_mock() + insert_datetime_mock() + flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) + + module.create_archive( + excludes_filename=None, + verbosity=None, + storage_config={}, + source_directories='foo*', repository='repo', command='attic', ) From 15bf273e6e998e4064a3d7009d404ee660c549a8 Mon Sep 17 00:00:00 2001 From: Jan Gondol Date: Wed, 6 Apr 2016 14:54:06 +0200 Subject: [PATCH 093/189] Fix broken link to Borg quickstart --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1082ff8..febacf0 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on To get up and running, follow the [Attic Quick Start](https://attic-backup.org/quickstart.html) or the [Borg Quick -Start](https://borgbackup.github.io/borgbackup/quickstart.html) to create a +Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create a repository on a local or remote host. Note that if you plan to run atticmatic on a schedule with cron, and you encrypt your attic repository with a passphrase instead of a key file, you'll need to set the atticmatic From c3b4cb21ed77d5477026ec59d91c2ace18ee462e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 10 Apr 2016 10:23:32 -0700 Subject: [PATCH 094/189] Fixed links to Borg documentation. --- NEWS | 1 + README.md | 2 +- sample/config | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 77042d9..a2e7afa 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ * Added support for --one-file-system for Borg. * Support borg create --umask. * Added support for file globs in source_directories. + * Fixed links to Borg documentation. 0.1.7 diff --git a/README.md b/README.md index febacf0..aef1078 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ title: Atticmatic atticmatic is a simple Python wrapper script for the [Attic](https://attic-backup.org/) and -[Borg](https://borgbackup.github.io/borgbackup/) backup software that +[Borg](https://borgbackup.readthedocs.org/en/stable/) backup software that initiates a backup, prunes any old backups according to a retention policy, and validates backups for consistency. The script supports specifying your settings in a declarative configuration file rather than having to put them diff --git a/sample/config b/sample/config index e681376..5a4c2e2 100644 --- a/sample/config +++ b/sample/config @@ -15,7 +15,7 @@ repository: user@backupserver:sourcehostname.attic # were initialized with passphrase/repokey encryption. #encryption_passphrase: foo # For Borg only, you can specify the type of compression to use when creating -# archives. See https://borgbackup.github.io/borgbackup/usage.html#borg-create +# archives. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create # for details. Defaults to no compression. #compression: lz4 # For Borg only, you can specify the umask to be used for borg create. @@ -24,7 +24,7 @@ repository: user@backupserver:sourcehostname.attic [retention] # Retention policy for how many backups to keep in each category. See # https://attic-backup.org/usage.html#attic-prune or -# https://borgbackup.github.io/borgbackup/usage.html#borg-prune for details. +# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details. #keep_within: 3H #keep_hourly: 24 keep_daily: 7 @@ -37,7 +37,7 @@ keep_yearly: 1 # Space-separated list of consistency checks to run: "repository", "archives", # or both. Defaults to both. Set to "disabled" to disable all consistency # checks. See https://attic-backup.org/usage.html#attic-check or -# https://borgbackup.github.io/borgbackup/usage.html#borg-check for details. +# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. checks: repository archives # For Borg only, you can restrict the number of checked archives to the last n. #check_last: 3 From fa87aed263b163bb3e79689fce8390379ab9576e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 10 Apr 2016 15:27:21 -0700 Subject: [PATCH 095/189] Normalizing recent changes. No new content. --- NEWS | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index a2e7afa..ddcfe78 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,10 @@ 0.1.8.dev0 - * Fixed handling of repeated spaces in source_directories which resulted in backup up everything. - * Added support for --one-file-system for Borg. - * Support borg create --umask. - * Added support for file globs in source_directories. - * Fixed links to Borg documentation. + * Fix for handling of spaces in source_directories which resulted in backup up everything. + * Fix for broken links to Borg documentation. + * Support for Borg --one-file-system. + * Support for Borg create --umask. + * support for file globs in source_directories. 0.1.7 From 82e8dae94899c6305bacafb2213c130f61c36dfa Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 10 Apr 2016 15:29:42 -0700 Subject: [PATCH 096/189] At verbosity zero, suppressing Borg check spew to stderr about "Checking segments". --- NEWS | 3 ++- atticmatic/backends/shared.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index ddcfe78..321eb6f 100644 --- a/NEWS +++ b/NEWS @@ -2,9 +2,10 @@ * Fix for handling of spaces in source_directories which resulted in backup up everything. * Fix for broken links to Borg documentation. + * At verbosity zero, suppressing Borg check spew to stderr about "Checking segments". * Support for Borg --one-file-system. * Support for Borg create --umask. - * support for file globs in source_directories. + * Support for file globs in source_directories. 0.1.7 diff --git a/atticmatic/backends/shared.py b/atticmatic/backends/shared.py index 417a732..2fe6a5c 100644 --- a/atticmatic/backends/shared.py +++ b/atticmatic/backends/shared.py @@ -213,7 +213,7 @@ def check_archives(verbosity, repository, consistency_config, command): repository, ) + _make_check_flags(checks, check_last) + verbosity_flags - # The check command spews to stdout even without the verbose flag. Suppress it. + # The check command spews to stdout/stderr even without the verbose flag. Suppress it. stdout = None if verbosity_flags else open(os.devnull, 'w') - subprocess.check_call(full_command, stdout=stdout) + subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) From 8210172d7f8317f45705255f1e353d5b9c2e6697 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 10 Apr 2016 15:46:43 -0700 Subject: [PATCH 097/189] Fixing "check" backend tests to support new use of stderr=STDOUT. --- atticmatic/tests/unit/backends/test_shared.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/atticmatic/tests/unit/backends/test_shared.py b/atticmatic/tests/unit/backends/test_shared.py index cddef00..26c8b0d 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/atticmatic/tests/unit/backends/test_shared.py @@ -1,4 +1,5 @@ from collections import OrderedDict +from subprocess import STDOUT import os from flexmock import flexmock @@ -31,7 +32,7 @@ def test_initialize_without_passphrase_should_not_set_environment(): def insert_subprocess_mock(check_call_command, **kwargs): - subprocess = flexmock() + subprocess = flexmock(STDOUT=STDOUT) subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() flexmock(module).subprocess = subprocess @@ -353,7 +354,7 @@ def test_check_archives_should_call_attic_with_parameters(): stdout = flexmock() insert_subprocess_mock( ('attic', 'check', 'repo'), - stdout=stdout, + stdout=stdout, stderr=STDOUT, ) insert_platform_mock() insert_datetime_mock() @@ -374,7 +375,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( ('attic', 'check', 'repo', '--verbose'), - stdout=None, + stdout=None, stderr=STDOUT, ) insert_platform_mock() insert_datetime_mock() @@ -393,7 +394,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( ('attic', 'check', 'repo', '--verbose'), - stdout=None, + stdout=None, stderr=STDOUT, ) insert_platform_mock() insert_datetime_mock() From 5b66dc69a1b262bed928567205188f634c3e9573 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 10 Apr 2016 15:48:10 -0700 Subject: [PATCH 098/189] Refreshing flexmock version in test requirements. --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 5a34a85..8969b43 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,2 +1,2 @@ -flexmock==0.9.7 +flexmock==0.10.2 nose==1.3.4 From 0ea58244272e74ed4a083686f67f74e570559c8c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 10 Apr 2016 15:59:36 -0700 Subject: [PATCH 099/189] Switching from the no-longer-maintained nose test runner to pytest. --- atticmatic/tests/integration/test_command.py | 4 ++-- atticmatic/tests/unit/test_config.py | 12 ++++++------ setup.cfg | 3 --- setup.py | 2 +- test_requirements.txt | 2 +- tox.ini | 2 +- 6 files changed, 11 insertions(+), 14 deletions(-) diff --git a/atticmatic/tests/integration/test_command.py b/atticmatic/tests/integration/test_command.py index 346612b..b26d815 100644 --- a/atticmatic/tests/integration/test_command.py +++ b/atticmatic/tests/integration/test_command.py @@ -2,7 +2,7 @@ import os import sys from flexmock import flexmock -from nose.tools import assert_raises +import pytest from atticmatic import command as module @@ -66,7 +66,7 @@ def test_parse_arguments_with_invalid_arguments_exits(): sys.stderr = sys.stdout try: - with assert_raises(SystemExit): + with pytest.raises(SystemExit): module.parse_arguments(COMMAND_NAME, '--posix-me-harder') finally: sys.stderr = original_stderr diff --git a/atticmatic/tests/unit/test_config.py b/atticmatic/tests/unit/test_config.py index 88569e1..6422fa5 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/atticmatic/tests/unit/test_config.py @@ -1,7 +1,7 @@ from collections import OrderedDict from flexmock import flexmock -from nose.tools import assert_raises +import pytest from atticmatic import config as module @@ -61,7 +61,7 @@ def test_validate_configuration_format_with_missing_required_section_should_rais ), ) - with assert_raises(ValueError): + with pytest.raises(ValueError): module.validate_configuration_format(parser, config_format) @@ -96,7 +96,7 @@ def test_validate_configuration_format_with_unknown_section_should_raise(): module.Section_format('section', options=()), ) - with assert_raises(ValueError): + with pytest.raises(ValueError): module.validate_configuration_format(parser, config_format) @@ -114,7 +114,7 @@ def test_validate_configuration_format_with_missing_required_option_should_raise ), ) - with assert_raises(ValueError): + with pytest.raises(ValueError): module.validate_configuration_format(parser, config_format) @@ -146,7 +146,7 @@ def test_validate_configuration_format_with_extra_option_should_raise(): ), ) - with assert_raises(ValueError): + with pytest.raises(ValueError): module.validate_configuration_format(parser, config_format) @@ -228,5 +228,5 @@ def test_parse_configuration_with_file_open_error_should_raise(): parser = insert_mock_parser() parser.should_receive('read').and_return([]) - with assert_raises(ValueError): + with pytest.raises(ValueError): module.parse_configuration('filename', config_format=flexmock()) diff --git a/setup.cfg b/setup.cfg index 222a3da..12871ff 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,2 @@ [metadata] description-file=README.md - -[nosetests] -detailed-errors=1 diff --git a/setup.py b/setup.py index ff5a4be..f399973 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,6 @@ setup( }, tests_require=( 'flexmock', - 'nose', + 'pytest', ) ) diff --git a/test_requirements.txt b/test_requirements.txt index 8969b43..3c5f9e5 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,2 +1,2 @@ flexmock==0.10.2 -nose==1.3.4 +pytest==2.9.1 diff --git a/tox.ini b/tox.ini index 6ac577b..64d1f6c 100644 --- a/tox.ini +++ b/tox.ini @@ -5,4 +5,4 @@ skipsdist=True [testenv] usedevelop=True deps=-rtest_requirements.txt -commands = nosetests [] +commands = py.test [] From 9e45da75cb642a1df4a4a381070c63931851fa53 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 10 Apr 2016 16:01:05 -0700 Subject: [PATCH 100/189] Cutting a release. --- NEWS | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 321eb6f..8e26a42 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,8 @@ -0.1.8.dev0 +0.1.8 * Fix for handling of spaces in source_directories which resulted in backup up everything. * Fix for broken links to Borg documentation. - * At verbosity zero, suppressing Borg check spew to stderr about "Checking segments". + * At verbosity zero, suppressing Borg check stderr spew about "Checking segments". * Support for Borg --one-file-system. * Support for Borg create --umask. * Support for file globs in source_directories. diff --git a/setup.py b/setup.py index f399973..9acda9e 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '0.1.8.dev0' +VERSION = '0.1.8' setup( From c7e23fe9edb71fa93762db0190a67af19d552498 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 10 Apr 2016 16:01:18 -0700 Subject: [PATCH 101/189] Added tag 0.1.8 for changeset dbc96d3f83bd --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index b580959..65c26a7 100644 --- a/.hgtags +++ b/.hgtags @@ -25,3 +25,4 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4 977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files 0000000000000000000000000000000000000000 github/yaml_config_files 28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master +dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8 From 175761c7575ccef848f0638a1e5b8b816c9961a9 Mon Sep 17 00:00:00 2001 From: dawez Date: Sun, 17 Apr 2016 22:26:07 +0200 Subject: [PATCH 102/189] fixed typos in readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aef1078..59bf40c 100644 --- a/README.md +++ b/README.md @@ -93,15 +93,15 @@ By default, the backup will proceed silently except in the case of errors. But if you'd like to to get additional information about the progress of the backup as it proceeds, use the verbosity option: - atticmattic --verbosity 1 + atticmatic --verbosity 1 Or, for even more progress spew: - atticmattic --verbosity 2 + atticmatic --verbosity 2 If you'd like to see the available command-line arguments, view the help: - atticmattic --help + atticmatic --help ## Running tests From 633700c0af85eb41ec143f831fed0678abae2d37 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 10 Jun 2016 11:21:53 -0700 Subject: [PATCH 103/189] Dropping support for Attic. --- NEWS | 6 ++ README.md | 61 ++++++--------- atticmatic/backends/attic.py | 14 ---- atticmatic/backends/borg.py | 41 ---------- atticmatic/command.py | 75 ------------------- atticmatic/tests/unit/__init__.py | 0 atticmatic/tests/unit/backends/__init__.py | 0 atticmatic/tests/unit/test_command.py | 33 -------- {atticmatic => borgmatic}/__init__.py | 0 .../backends/shared.py => borgmatic/borg.py | 50 ++----------- borgmatic/command.py | 57 ++++++++++++++ {atticmatic => borgmatic}/config.py | 39 ++++++++++ .../backends => borgmatic/tests}/__init__.py | 0 {atticmatic => borgmatic}/tests/builtins.py | 0 .../tests/integration}/__init__.py | 0 .../tests/integration/test_command.py | 29 ++++--- .../tests/integration/test_config.py | 2 +- .../tests/integration/test_version.py | 0 .../tests/unit}/__init__.py | 0 .../tests/unit/test_borg.py | 6 +- .../tests/unit/test_config.py | 2 +- {atticmatic => borgmatic}/verbosity.py | 0 sample/atticmatic.cron | 3 - sample/config | 15 ++-- setup.py | 16 ++-- 25 files changed, 167 insertions(+), 282 deletions(-) delete mode 100644 atticmatic/backends/attic.py delete mode 100644 atticmatic/backends/borg.py delete mode 100644 atticmatic/command.py delete mode 100644 atticmatic/tests/unit/__init__.py delete mode 100644 atticmatic/tests/unit/backends/__init__.py delete mode 100644 atticmatic/tests/unit/test_command.py rename {atticmatic => borgmatic}/__init__.py (100%) rename atticmatic/backends/shared.py => borgmatic/borg.py (80%) create mode 100644 borgmatic/command.py rename {atticmatic => borgmatic}/config.py (80%) rename {atticmatic/backends => borgmatic/tests}/__init__.py (100%) rename {atticmatic => borgmatic}/tests/builtins.py (100%) rename {atticmatic/tests => borgmatic/tests/integration}/__init__.py (100%) rename {atticmatic => borgmatic}/tests/integration/test_command.py (72%) rename {atticmatic => borgmatic}/tests/integration/test_config.py (94%) rename {atticmatic => borgmatic}/tests/integration/test_version.py (100%) rename {atticmatic/tests/integration => borgmatic/tests/unit}/__init__.py (100%) rename atticmatic/tests/unit/backends/test_shared.py => borgmatic/tests/unit/test_borg.py (98%) rename {atticmatic => borgmatic}/tests/unit/test_config.py (99%) rename {atticmatic => borgmatic}/verbosity.py (100%) delete mode 100644 sample/atticmatic.cron diff --git a/NEWS b/NEWS index 8e26a42..312fc1f 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,9 @@ +1.0.0 + + * Attic is no longer supported, as there hasn't been any recent development on it. + This will allow faster iteration on Borg-specific configuration. + * Project renamed from atticmatic to borgmatic. + 0.1.8 * Fix for handling of spaces in source_directories which resulted in backup up everything. diff --git a/README.md b/README.md index 59bf40c..71a9300 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ -title: Atticmatic +title: Borgmatic ## Overview -atticmatic is a simple Python wrapper script for the -[Attic](https://attic-backup.org/) and +borgmatic (formerly atticmatic) is a simple Python wrapper script for the [Borg](https://borgbackup.readthedocs.org/en/stable/) backup software that initiates a backup, prunes any old backups according to a retention policy, and validates backups for consistency. The script supports specifying your @@ -19,7 +18,7 @@ Here's an example config file: source_directories: /home /etc /var/log/syslog* # Path to local or remote backup repository. -repository: user@backupserver:sourcehostname.attic +repository: user@backupserver:sourcehostname.borg [retention] # Retention policy for how many backups to keep in each category. @@ -35,37 +34,30 @@ checks: repository archives Additionally, exclude patterns can be specified in a separate excludes config file, one pattern per line. -atticmatic is hosted at with [source code -available](https://torsion.org/hg/atticmatic). It's also mirrored on -[GitHub](https://github.com/witten/atticmatic) and -[BitBucket](https://bitbucket.org/dhelfman/atticmatic) for convenience. +borgmatic is hosted at with [source code +available](https://torsion.org/hg/borgmatic). It's also mirrored on +[GitHub](https://github.com/witten/borgmatic) and +[BitBucket](https://bitbucket.org/dhelfman/borgmatic) for convenience. ## Setup -To get up and running, follow the [Attic Quick -Start](https://attic-backup.org/quickstart.html) or the [Borg Quick -Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create a -repository on a local or remote host. Note that if you plan to run atticmatic -on a schedule with cron, and you encrypt your attic repository with a -passphrase instead of a key file, you'll need to set the atticmatic +To get up and running, follow the [Borg Quick +Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create +a repository on a local or remote host. Note that if you plan to run +borgmatic on a schedule with cron, and you encrypt your Borg repository with +a passphrase instead of a key file, you'll need to set the borgmatic `encryption_passphrase` configuration variable. See the repository encryption section of the Quick Start for more info. If the repository is on a remote host, make sure that your local root user has key-based ssh access to the desired user account on the remote host. -To install atticmatic, run the following command to download and install it: +To install borgmatic, run the following command to download and install it: - sudo pip install --upgrade atticmatic + sudo pip install --upgrade borgmatic -If you are using Attic, copy the following configuration files: - - sudo cp sample/atticmatic.cron /etc/cron.d/atticmatic - sudo mkdir /etc/atticmatic/ - sudo cp sample/config sample/excludes /etc/atticmatic/ - -If you are using Borg, copy the files like this instead: +Then, copy the following configuration files: sudo cp sample/borgmatic.cron /etc/cron.d/borgmatic sudo mkdir /etc/borgmatic/ @@ -76,14 +68,9 @@ Lastly, modify the /etc files with your desired configuration. ## Usage -You can run atticmatic and start a backup simply by invoking it without +You can run borgmatic and start a backup simply by invoking it without arguments: - atticmatic - -Or, if you're using Borg, use this command instead to make use of the Borg -backend: - borgmatic This will also prune any old backups as per the configured retention policy, @@ -93,15 +80,15 @@ By default, the backup will proceed silently except in the case of errors. But if you'd like to to get additional information about the progress of the backup as it proceeds, use the verbosity option: - atticmatic --verbosity 1 + borgmatic --verbosity 1 Or, for even more progress spew: - atticmatic --verbosity 2 + borgmatic --verbosity 2 If you'd like to see the available command-line arguments, view the help: - atticmatic --help + borgmatic --help ## Running tests @@ -119,12 +106,12 @@ Then, to actually run tests, run: ### Broken pipe with remote repository -When running atticmatic on a large remote repository, you may receive errors -like the following, particularly while "attic check" is validating backups for +When running borgmatic on a large remote repository, you may receive errors +like the following, particularly while "borg check" is validating backups for consistency: Write failed: Broken pipe - attic: Error: Connection closed by remote host + borg: Error: Connection closed by remote host This error can be caused by an ssh timeout, which you can rectify by adding the following to the ~/.ssh/config file on the client: @@ -138,8 +125,8 @@ backups. ## Issues and feedback -Got an issue or an idea for a feature enhancement? Check out the [atticmatic -issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues). In +Got an issue or an idea for a feature enhancement? Check out the [borgmatic +issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues). In order to create a new issue or comment on an issue, you'll need to [login first](https://tree.taiga.io/login). diff --git a/atticmatic/backends/attic.py b/atticmatic/backends/attic.py deleted file mode 100644 index daa0b29..0000000 --- a/atticmatic/backends/attic.py +++ /dev/null @@ -1,14 +0,0 @@ -from functools import partial - -from atticmatic.backends import shared - -# An atticmatic backend that supports Attic for actually handling backups. - -COMMAND = 'attic' -CONFIG_FORMAT = shared.CONFIG_FORMAT - - -initialize = partial(shared.initialize, command=COMMAND) -create_archive = partial(shared.create_archive, command=COMMAND) -prune_archives = partial(shared.prune_archives, command=COMMAND) -check_archives = partial(shared.check_archives, command=COMMAND) diff --git a/atticmatic/backends/borg.py b/atticmatic/backends/borg.py deleted file mode 100644 index d99af4a..0000000 --- a/atticmatic/backends/borg.py +++ /dev/null @@ -1,41 +0,0 @@ -from functools import partial - -from atticmatic.config import Section_format, option -from atticmatic.backends import shared - -# An atticmatic backend that supports Borg for actually handling backups. - -COMMAND = 'borg' -CONFIG_FORMAT = ( - Section_format( - 'location', - ( - option('source_directories'), - option('one_file_system', value_type=bool, required=False), - option('repository'), - ), - ), - Section_format( - 'storage', - ( - option('encryption_passphrase', required=False), - option('compression', required=False), - option('umask', required=False), - ), - ), - shared.CONFIG_FORMAT[2], # retention - Section_format( - 'consistency', - ( - option('checks', required=False), - option('check_last', required=False), - ), - ) -) - - -initialize = partial(shared.initialize, command=COMMAND) - -create_archive = partial(shared.create_archive, command=COMMAND) -prune_archives = partial(shared.prune_archives, command=COMMAND) -check_archives = partial(shared.check_archives, command=COMMAND) diff --git a/atticmatic/command.py b/atticmatic/command.py deleted file mode 100644 index 08ea49e..0000000 --- a/atticmatic/command.py +++ /dev/null @@ -1,75 +0,0 @@ -from __future__ import print_function -from argparse import ArgumentParser -from importlib import import_module -import os -from subprocess import CalledProcessError -import sys - -from atticmatic.config import parse_configuration - - -DEFAULT_CONFIG_FILENAME_PATTERN = '/etc/{}/config' -DEFAULT_EXCLUDES_FILENAME_PATTERN = '/etc/{}/excludes' - - -def parse_arguments(command_name, *arguments): - ''' - Given the name of the command with which this script was invoked and command-line arguments, - parse the arguments and return them as an ArgumentParser instance. Use the command name to - determine the default configuration and excludes paths. - ''' - config_filename_default = DEFAULT_CONFIG_FILENAME_PATTERN.format(command_name) - excludes_filename_default = DEFAULT_EXCLUDES_FILENAME_PATTERN.format(command_name) - - parser = ArgumentParser() - parser.add_argument( - '-c', '--config', - dest='config_filename', - default=config_filename_default, - help='Configuration filename', - ) - parser.add_argument( - '--excludes', - dest='excludes_filename', - default=excludes_filename_default if os.path.exists(excludes_filename_default) else None, - help='Excludes filename', - ) - parser.add_argument( - '-v', '--verbosity', - type=int, - help='Display verbose progress (1 for some, 2 for lots)', - ) - - return parser.parse_args(arguments) - - -def load_backend(command_name): - ''' - Given the name of the command with which this script was invoked, return the corresponding - backend module responsible for actually dealing with backups. - ''' - backend_name = { - 'atticmatic': 'attic', - 'borgmatic': 'borg', - }.get(command_name, 'attic') - - return import_module('atticmatic.backends.{}'.format(backend_name)) - - -def main(): - try: - command_name = os.path.basename(sys.argv[0]) - args = parse_arguments(command_name, *sys.argv[1:]) - backend = load_backend(command_name) - config = parse_configuration(args.config_filename, backend.CONFIG_FORMAT) - repository = config.location['repository'] - - backend.initialize(config.storage) - backend.create_archive( - args.excludes_filename, args.verbosity, config.storage, **config.location - ) - backend.prune_archives(args.verbosity, repository, config.retention) - backend.check_archives(args.verbosity, repository, config.consistency) - except (ValueError, IOError, CalledProcessError) as error: - print(error, file=sys.stderr) - sys.exit(1) diff --git a/atticmatic/tests/unit/__init__.py b/atticmatic/tests/unit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/atticmatic/tests/unit/backends/__init__.py b/atticmatic/tests/unit/backends/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/atticmatic/tests/unit/test_command.py b/atticmatic/tests/unit/test_command.py deleted file mode 100644 index 6a3cdb1..0000000 --- a/atticmatic/tests/unit/test_command.py +++ /dev/null @@ -1,33 +0,0 @@ -from flexmock import flexmock - -from atticmatic import command as module - - -def test_load_backend_with_atticmatic_command_should_return_attic_backend(): - backend = flexmock() - ( - flexmock(module).should_receive('import_module').with_args('atticmatic.backends.attic') - .and_return(backend).once() - ) - - assert module.load_backend('atticmatic') == backend - - -def test_load_backend_with_unknown_command_should_return_attic_backend(): - backend = flexmock() - ( - flexmock(module).should_receive('import_module').with_args('atticmatic.backends.attic') - .and_return(backend).once() - ) - - assert module.load_backend('unknownmatic') == backend - - -def test_load_backend_with_borgmatic_command_should_return_borg_backend(): - backend = flexmock() - ( - flexmock(module).should_receive('import_module').with_args('atticmatic.backends.borg') - .and_return(backend).once() - ) - - assert module.load_backend('borgmatic') == backend diff --git a/atticmatic/__init__.py b/borgmatic/__init__.py similarity index 100% rename from atticmatic/__init__.py rename to borgmatic/__init__.py diff --git a/atticmatic/backends/shared.py b/borgmatic/borg.py similarity index 80% rename from atticmatic/backends/shared.py rename to borgmatic/borg.py index 2fe6a5c..adde7cf 100644 --- a/atticmatic/backends/shared.py +++ b/borgmatic/borg.py @@ -6,52 +6,16 @@ import subprocess from glob import glob from itertools import chain -from atticmatic.config import Section_format, option -from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS +from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS -# Common backend functionality shared by Attic and Borg. As the two backup -# commands diverge, these shared functions will likely need to be replaced -# with non-shared functions within atticmatic.backends.attic and -# atticmatic.backends.borg. +# Integration with Borg for actually handling backups. -CONFIG_FORMAT = ( - Section_format( - 'location', - ( - option('source_directories'), - option('repository'), - ), - ), - Section_format( - 'storage', - ( - option('encryption_passphrase', required=False), - ), - ), - Section_format( - 'retention', - ( - option('keep_within', required=False), - option('keep_hourly', int, required=False), - option('keep_daily', int, required=False), - option('keep_weekly', int, required=False), - option('keep_monthly', int, required=False), - option('keep_yearly', int, required=False), - option('prefix', required=False), - ), - ), - Section_format( - 'consistency', - ( - option('checks', required=False), - ), - ) -) +COMMAND = 'borg' -def initialize(storage_config, command): +def initialize(storage_config, command=COMMAND): passphrase = storage_config.get('encryption_passphrase') if passphrase: @@ -59,7 +23,7 @@ def initialize(storage_config, command): def create_archive( - excludes_filename, verbosity, storage_config, source_directories, repository, command, + excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND, one_file_system=None ): ''' @@ -115,7 +79,7 @@ def _make_prune_flags(retention_config): ) -def prune_archives(verbosity, repository, retention_config, command): +def prune_archives(verbosity, repository, retention_config, command=COMMAND): ''' Given a verbosity flag, a local or remote repository path, a retention config dict, and a command to run, prune attic archives according the the retention policy specified in that @@ -191,7 +155,7 @@ def _make_check_flags(checks, check_last=None): ) + last_flag -def check_archives(verbosity, repository, consistency_config, command): +def check_archives(verbosity, repository, consistency_config, command=COMMAND): ''' Given a verbosity flag, a local or remote repository path, a consistency config dict, and a command to run, check the contained attic archives for consistency. diff --git a/borgmatic/command.py b/borgmatic/command.py new file mode 100644 index 0000000..337fc3f --- /dev/null +++ b/borgmatic/command.py @@ -0,0 +1,57 @@ +from __future__ import print_function +from argparse import ArgumentParser +import os +from subprocess import CalledProcessError +import sys + +from borgmatic import borg +from borgmatic.config import parse_configuration, CONFIG_FORMAT + + +DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config' +DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' + + +def parse_arguments(*arguments): + ''' + Given the name of the command with which this script was invoked and command-line arguments, + parse the arguments and return them as an ArgumentParser instance. Use the command name to + determine the default configuration and excludes paths. + ''' + parser = ArgumentParser() + parser.add_argument( + '-c', '--config', + dest='config_filename', + default=DEFAULT_CONFIG_FILENAME, + help='Configuration filename', + ) + parser.add_argument( + '--excludes', + dest='excludes_filename', + default=DEFAULT_EXCLUDES_FILENAME if os.path.exists(DEFAULT_EXCLUDES_FILENAME) else None, + help='Excludes filename', + ) + parser.add_argument( + '-v', '--verbosity', + type=int, + help='Display verbose progress (1 for some, 2 for lots)', + ) + + return parser.parse_args(arguments) + + +def main(): + try: + args = parse_arguments(*sys.argv[1:]) + config = parse_configuration(args.config_filename, CONFIG_FORMAT) + repository = config.location['repository'] + + borg.initialize(config.storage) + borg.create_archive( + args.excludes_filename, args.verbosity, config.storage, **config.location + ) + borg.prune_archives(args.verbosity, repository, config.retention) + borg.check_archives(args.verbosity, repository, config.consistency) + except (ValueError, IOError, CalledProcessError) as error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/atticmatic/config.py b/borgmatic/config.py similarity index 80% rename from atticmatic/config.py rename to borgmatic/config.py index 8f7ae9a..d7a9a08 100644 --- a/atticmatic/config.py +++ b/borgmatic/config.py @@ -20,6 +20,45 @@ def option(name, value_type=str, required=True): return Config_option(name, value_type, required) +CONFIG_FORMAT = ( + Section_format( + 'location', + ( + option('source_directories'), + option('one_file_system', value_type=bool, required=False), + option('repository'), + ), + ), + Section_format( + 'storage', + ( + option('encryption_passphrase', required=False), + option('compression', required=False), + option('umask', required=False), + ), + ), + Section_format( + 'retention', + ( + option('keep_within', required=False), + option('keep_hourly', int, required=False), + option('keep_daily', int, required=False), + option('keep_weekly', int, required=False), + option('keep_monthly', int, required=False), + option('keep_yearly', int, required=False), + option('prefix', required=False), + ), + ), + Section_format( + 'consistency', + ( + option('checks', required=False), + option('check_last', required=False), + ), + ) +) + + def validate_configuration_format(parser, config_format): ''' Given an open RawConfigParser and an expected config file format, validate that the parsed diff --git a/atticmatic/backends/__init__.py b/borgmatic/tests/__init__.py similarity index 100% rename from atticmatic/backends/__init__.py rename to borgmatic/tests/__init__.py diff --git a/atticmatic/tests/builtins.py b/borgmatic/tests/builtins.py similarity index 100% rename from atticmatic/tests/builtins.py rename to borgmatic/tests/builtins.py diff --git a/atticmatic/tests/__init__.py b/borgmatic/tests/integration/__init__.py similarity index 100% rename from atticmatic/tests/__init__.py rename to borgmatic/tests/integration/__init__.py diff --git a/atticmatic/tests/integration/test_command.py b/borgmatic/tests/integration/test_command.py similarity index 72% rename from atticmatic/tests/integration/test_command.py rename to borgmatic/tests/integration/test_command.py index b26d815..c1e311c 100644 --- a/atticmatic/tests/integration/test_command.py +++ b/borgmatic/tests/integration/test_command.py @@ -4,26 +4,23 @@ import sys from flexmock import flexmock import pytest -from atticmatic import command as module - - -COMMAND_NAME = 'foomatic' +from borgmatic import command as module def test_parse_arguments_with_no_arguments_uses_defaults(): flexmock(os.path).should_receive('exists').and_return(True) - parser = module.parse_arguments(COMMAND_NAME) + parser = module.parse_arguments() - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME) - assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME_PATTERN.format(COMMAND_NAME) + assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME + assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME assert parser.verbosity == None def test_parse_arguments_with_filename_arguments_overrides_defaults(): flexmock(os.path).should_receive('exists').and_return(True) - parser = module.parse_arguments(COMMAND_NAME, '--config', 'myconfig', '--excludes', 'myexcludes') + parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes') assert parser.config_filename == 'myconfig' assert parser.excludes_filename == 'myexcludes' @@ -33,9 +30,9 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults(): def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none(): flexmock(os.path).should_receive('exists').and_return(False) - parser = module.parse_arguments(COMMAND_NAME) + parser = module.parse_arguments() - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME) + assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME assert parser.excludes_filename == None assert parser.verbosity == None @@ -43,9 +40,9 @@ def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_non def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename(): flexmock(os.path).should_receive('exists').and_return(False) - parser = module.parse_arguments(COMMAND_NAME, '--excludes', 'myexcludes') + parser = module.parse_arguments('--excludes', 'myexcludes') - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME) + assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME assert parser.excludes_filename == 'myexcludes' assert parser.verbosity == None @@ -53,10 +50,10 @@ def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename( def test_parse_arguments_with_verbosity_flag_overrides_default(): flexmock(os.path).should_receive('exists').and_return(True) - parser = module.parse_arguments(COMMAND_NAME, '--verbosity', '1') + parser = module.parse_arguments('--verbosity', '1') - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME) - assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME_PATTERN.format(COMMAND_NAME) + assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME + assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME assert parser.verbosity == 1 @@ -67,6 +64,6 @@ def test_parse_arguments_with_invalid_arguments_exits(): try: with pytest.raises(SystemExit): - module.parse_arguments(COMMAND_NAME, '--posix-me-harder') + module.parse_arguments('--posix-me-harder') finally: sys.stderr = original_stderr diff --git a/atticmatic/tests/integration/test_config.py b/borgmatic/tests/integration/test_config.py similarity index 94% rename from atticmatic/tests/integration/test_config.py rename to borgmatic/tests/integration/test_config.py index 31fcd9e..e849b4f 100644 --- a/atticmatic/tests/integration/test_config.py +++ b/borgmatic/tests/integration/test_config.py @@ -8,7 +8,7 @@ except ImportError: from collections import OrderedDict import string -from atticmatic import config as module +from borgmatic import config as module def test_parse_section_options_with_punctuation_should_return_section_options(): diff --git a/atticmatic/tests/integration/test_version.py b/borgmatic/tests/integration/test_version.py similarity index 100% rename from atticmatic/tests/integration/test_version.py rename to borgmatic/tests/integration/test_version.py diff --git a/atticmatic/tests/integration/__init__.py b/borgmatic/tests/unit/__init__.py similarity index 100% rename from atticmatic/tests/integration/__init__.py rename to borgmatic/tests/unit/__init__.py diff --git a/atticmatic/tests/unit/backends/test_shared.py b/borgmatic/tests/unit/test_borg.py similarity index 98% rename from atticmatic/tests/unit/backends/test_shared.py rename to borgmatic/tests/unit/test_borg.py index 26c8b0d..1c032bb 100644 --- a/atticmatic/tests/unit/backends/test_shared.py +++ b/borgmatic/tests/unit/test_borg.py @@ -4,9 +4,9 @@ import os from flexmock import flexmock -from atticmatic.backends import shared as module -from atticmatic.tests.builtins import builtins_mock -from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS +from borgmatic import borg as module +from borgmatic.tests.builtins import builtins_mock +from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS def test_initialize_with_passphrase_should_set_environment(): diff --git a/atticmatic/tests/unit/test_config.py b/borgmatic/tests/unit/test_config.py similarity index 99% rename from atticmatic/tests/unit/test_config.py rename to borgmatic/tests/unit/test_config.py index 6422fa5..01e21bb 100644 --- a/atticmatic/tests/unit/test_config.py +++ b/borgmatic/tests/unit/test_config.py @@ -3,7 +3,7 @@ from collections import OrderedDict from flexmock import flexmock import pytest -from atticmatic import config as module +from borgmatic import config as module def test_option_should_create_config_option(): diff --git a/atticmatic/verbosity.py b/borgmatic/verbosity.py similarity index 100% rename from atticmatic/verbosity.py rename to borgmatic/verbosity.py diff --git a/sample/atticmatic.cron b/sample/atticmatic.cron deleted file mode 100644 index 39bc6bc..0000000 --- a/sample/atticmatic.cron +++ /dev/null @@ -1,3 +0,0 @@ -# You can drop this file into /etc/cron.d/ to run atticmatic nightly. - -0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/atticmatic diff --git a/sample/config b/sample/config index 5a4c2e2..acea8d7 100644 --- a/sample/config +++ b/sample/config @@ -8,22 +8,21 @@ source_directories: /home /etc /var/log/syslog* #one_file_system: True # Path to local or remote repository. -repository: user@backupserver:sourcehostname.attic +repository: user@backupserver:sourcehostname.borg [storage] # Passphrase to unlock the encryption key with. Only use on repositories that # were initialized with passphrase/repokey encryption. #encryption_passphrase: foo -# For Borg only, you can specify the type of compression to use when creating -# archives. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create +# Type of compression to use when creating archives. See +# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create # for details. Defaults to no compression. #compression: lz4 -# For Borg only, you can specify the umask to be used for borg create. +# Umask to be used for borg create. #umask: 0740 [retention] # Retention policy for how many backups to keep in each category. See -# https://attic-backup.org/usage.html#attic-prune or # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details. #keep_within: 3H #keep_hourly: 24 @@ -36,8 +35,8 @@ keep_yearly: 1 [consistency] # Space-separated list of consistency checks to run: "repository", "archives", # or both. Defaults to both. Set to "disabled" to disable all consistency -# checks. See https://attic-backup.org/usage.html#attic-check or -# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. +# checks. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check +# for details. checks: repository archives -# For Borg only, you can restrict the number of checked archives to the last n. +# Restrict the number of checked archives to the last n. #check_last: 3 diff --git a/setup.py b/setup.py index 9acda9e..f933976 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,17 @@ from setuptools import setup, find_packages -VERSION = '0.1.8' +VERSION = '1.0.0' setup( - name='atticmatic', + name='borgmatic', version=VERSION, - description='A wrapper script for Attic/Borg backup software that creates and prunes backups', + description='A wrapper script for Borg backup software that creates and prunes backups', author='Dan Helfman', author_email='witten@torsion.org', - url='https://torsion.org/atticmatic', - download_url='https://torsion.org/hg/atticmatic/archive/%s.tar.gz' % VERSION, + url='https://torsion.org/borgmatic', + download_url='https://torsion.org/hg/borgmatic/archive/%s.tar.gz' % VERSION, classifiers=( 'Development Status :: 5 - Production/Stable', 'Environment :: Console', @@ -24,10 +24,12 @@ setup( packages=find_packages(), entry_points={ 'console_scripts': [ - 'atticmatic = atticmatic.command:main', - 'borgmatic = atticmatic.command:main', + 'borgmatic = borgmatic.command:main', ] }, + obsoletes=( + 'atticmatic', + ), tests_require=( 'flexmock', 'pytest', From 4533fec167eeb4c9a0560c24ea5909834920604d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 10 Jun 2016 11:53:45 -0700 Subject: [PATCH 104/189] Documenting how to upgrade from atticmatic to borgmatic. --- NEWS | 8 +++++--- README.md | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 312fc1f..7b7dccb 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,10 @@ 1.0.0 - * Attic is no longer supported, as there hasn't been any recent development on it. - This will allow faster iteration on Borg-specific configuration. - * Project renamed from atticmatic to borgmatic. + * Attic is no longer supported, as there hasn't been any recent development on + it. Dropping Attic support will allow faster iteration on Borg-specific + features. If you're still using Attic, this is a good time to switch to Borg! + * Project renamed from atticmatic to borgmatic. See the borgmatic README for + information on upgrading. 0.1.8 diff --git a/README.md b/README.md index 71a9300..b9e1d93 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,26 @@ Then, copy the following configuration files: Lastly, modify the /etc files with your desired configuration. +## Upgrading from atticmatic + +You can ignore this section if you're not an atticmatic user (the former name +of borgmatic). + +borgmatic only supports Borg now and no longer supports Attic. So if you're +an Attic user, consider switching to Borg. See the [Borg upgrade +command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade) +for more information. Then, follow the instructions above about setting up +your borgmatic configuration files. + +If you were already using Borg with atticmatic, then you can easily upgrade +from atticmatic to borgmatic. Simply run the following commands: + + sudo pip uninstall atticmatic + sudo pip install borgmatic + +That's it! borgmatic will continue using your /etc/borgmatic configuration +files. + ## Usage You can run borgmatic and start a backup simply by invoking it without From 377e3948ff4fba5a7f8dda820020f673447996c1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 10 Jun 2016 12:15:48 -0700 Subject: [PATCH 105/189] Added tag 1.0.0 for changeset 0e1fbee9358d --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 65c26a7..84d6b8e 100644 --- a/.hgtags +++ b/.hgtags @@ -26,3 +26,4 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4 0000000000000000000000000000000000000000 github/yaml_config_files 28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8 +0e1fbee9358de4f062fa9539e1355db83db70caa 1.0.0 From e1e5db22f8f38011c5aed57e09894eec2d1abecf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 10 Jun 2016 12:34:49 -0700 Subject: [PATCH 106/189] Making a univeral wheel that supports both Python 2 and 3. --- setup.cfg | 3 +++ setup.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 12871ff..4dd5142 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [metadata] description-file=README.md + +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py index f933976..19c1c60 100644 --- a/setup.py +++ b/setup.py @@ -27,9 +27,9 @@ setup( 'borgmatic = borgmatic.command:main', ] }, - obsoletes=( + obsoletes=[ 'atticmatic', - ), + ], tests_require=( 'flexmock', 'pytest', From 331adca23e49fa40e71efa2602b319334aef9307 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 10 Jun 2016 13:31:37 -0700 Subject: [PATCH 107/189] #19: Support for Borg's --remote-path option to use an alternate Borg executable. --- NEWS | 5 ++ borgmatic/borg.py | 15 ++-- borgmatic/command.py | 5 +- borgmatic/config.py | 1 + borgmatic/tests/unit/test_borg.py | 144 +++++++++++++++++++++--------- sample/config | 10 ++- setup.py | 2 +- 7 files changed, 127 insertions(+), 55 deletions(-) diff --git a/NEWS b/NEWS index 7b7dccb..af155a9 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.0.1 + + * #19: Support for Borg's --remote-path option to use an alternate Borg + executable. See sample/config. + 1.0.0 * Attic is no longer supported, as there hasn't been any recent development on diff --git a/borgmatic/borg.py b/borgmatic/borg.py index adde7cf..27c570a 100644 --- a/borgmatic/borg.py +++ b/borgmatic/borg.py @@ -24,7 +24,7 @@ def initialize(storage_config, command=COMMAND): def create_archive( excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND, - one_file_system=None + one_file_system=None, remote_path=None, ): ''' Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated @@ -39,6 +39,7 @@ def create_archive( umask = storage_config.get('umask', None) umask_flags = ('--umask', str(umask)) if umask else () one_file_system_flags = ('--one-file-system',) if one_file_system else () + remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { VERBOSITY_SOME: ('--stats',), VERBOSITY_LOTS: ('--verbose', '--stats'), @@ -52,7 +53,7 @@ def create_archive( timestamp=datetime.now().isoformat(), ), ) + sources + exclude_flags + compression_flags + one_file_system_flags + \ - umask_flags + verbosity_flags + remote_path_flags + umask_flags + verbosity_flags subprocess.check_call(full_command) @@ -79,12 +80,13 @@ def _make_prune_flags(retention_config): ) -def prune_archives(verbosity, repository, retention_config, command=COMMAND): +def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None): ''' Given a verbosity flag, a local or remote repository path, a retention config dict, and a command to run, prune attic archives according the the retention policy specified in that configuration. ''' + remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { VERBOSITY_SOME: ('--stats',), VERBOSITY_LOTS: ('--verbose', '--stats'), @@ -97,7 +99,7 @@ def prune_archives(verbosity, repository, retention_config, command=COMMAND): element for pair in _make_prune_flags(retention_config) for element in pair - ) + verbosity_flags + ) + remote_path_flags + verbosity_flags subprocess.check_call(full_command) @@ -155,7 +157,7 @@ def _make_check_flags(checks, check_last=None): ) + last_flag -def check_archives(verbosity, repository, consistency_config, command=COMMAND): +def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None): ''' Given a verbosity flag, a local or remote repository path, a consistency config dict, and a command to run, check the contained attic archives for consistency. @@ -167,6 +169,7 @@ def check_archives(verbosity, repository, consistency_config, command=COMMAND): if not checks: return + remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { VERBOSITY_SOME: ('--verbose',), VERBOSITY_LOTS: ('--verbose',), @@ -175,7 +178,7 @@ def check_archives(verbosity, repository, consistency_config, command=COMMAND): full_command = ( command, 'check', repository, - ) + _make_check_flags(checks, check_last) + verbosity_flags + ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags # The check command spews to stdout/stderr even without the verbose flag. Suppress it. stdout = None if verbosity_flags else open(os.devnull, 'w') diff --git a/borgmatic/command.py b/borgmatic/command.py index 337fc3f..bf6713d 100644 --- a/borgmatic/command.py +++ b/borgmatic/command.py @@ -45,13 +45,14 @@ def main(): args = parse_arguments(*sys.argv[1:]) config = parse_configuration(args.config_filename, CONFIG_FORMAT) repository = config.location['repository'] + remote_path = config.location['remote_path'] borg.initialize(config.storage) borg.create_archive( args.excludes_filename, args.verbosity, config.storage, **config.location ) - borg.prune_archives(args.verbosity, repository, config.retention) - borg.check_archives(args.verbosity, repository, config.consistency) + borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path) + borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path) except (ValueError, IOError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/borgmatic/config.py b/borgmatic/config.py index d7a9a08..36cb4f4 100644 --- a/borgmatic/config.py +++ b/borgmatic/config.py @@ -26,6 +26,7 @@ CONFIG_FORMAT = ( ( option('source_directories'), option('one_file_system', value_type=bool, required=False), + option('remote_path', required=False), option('repository'), ), ), diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 1c032bb..0511357 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -14,8 +14,8 @@ def test_initialize_with_passphrase_should_set_environment(): try: os.environ = {} - module.initialize({'encryption_passphrase': 'pass'}, command='attic') - assert os.environ.get('ATTIC_PASSPHRASE') == 'pass' + module.initialize({'encryption_passphrase': 'pass'}, command='borg') + assert os.environ.get('BORG_PASSPHRASE') == 'pass' finally: os.environ = orig_environ @@ -25,8 +25,8 @@ def test_initialize_without_passphrase_should_not_set_environment(): try: os.environ = {} - module.initialize({}, command='attic') - assert os.environ.get('ATTIC_PASSPHRASE') == None + module.initialize({}, command='borg') + assert os.environ.get('BORG_PASSPHRASE') == None finally: os.environ = orig_environ @@ -53,11 +53,11 @@ def insert_datetime_mock(): ).mock -CREATE_COMMAND_WITHOUT_EXCLUDES = ('attic', 'create', 'repo::host-now', 'foo', 'bar') +CREATE_COMMAND_WITHOUT_EXCLUDES = ('borg', 'create', 'repo::host-now', 'foo', 'bar') CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes') -def test_create_archive_should_call_attic_with_parameters(): +def test_create_archive_should_call_borg_with_parameters(): insert_subprocess_mock(CREATE_COMMAND) insert_platform_mock() insert_datetime_mock() @@ -68,7 +68,7 @@ def test_create_archive_should_call_attic_with_parameters(): storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) @@ -83,11 +83,11 @@ def test_create_archive_with_two_spaces_in_source_directories(): storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_none_excludes_filename_should_call_attic_without_excludes(): +def test_create_archive_with_none_excludes_filename_should_call_borg_without_excludes(): insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES) insert_platform_mock() insert_datetime_mock() @@ -98,11 +98,11 @@ def test_create_archive_with_none_excludes_filename_should_call_attic_without_ex storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter(): +def test_create_archive_with_verbosity_some_should_call_borg_with_stats_parameter(): insert_subprocess_mock(CREATE_COMMAND + ('--stats',)) insert_platform_mock() insert_datetime_mock() @@ -113,11 +113,11 @@ def test_create_archive_with_verbosity_some_should_call_attic_with_stats_paramet storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_parameter(): +def test_create_archive_with_verbosity_lots_should_call_borg_with_verbose_parameter(): insert_subprocess_mock(CREATE_COMMAND + ('--verbose', '--stats')) insert_platform_mock() insert_datetime_mock() @@ -128,11 +128,11 @@ def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_param storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_compression_should_call_attic_with_compression_parameters(): +def test_create_archive_with_compression_should_call_borg_with_compression_parameters(): insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) insert_platform_mock() insert_datetime_mock() @@ -143,11 +143,11 @@ def test_create_archive_with_compression_should_call_attic_with_compression_para storage_config={'compression': 'rle'}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_one_file_system_should_call_attic_with_one_file_system_parameters(): +def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters(): insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) insert_platform_mock() insert_datetime_mock() @@ -158,12 +158,28 @@ def test_create_archive_with_one_file_system_should_call_attic_with_one_file_sys storage_config={}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', one_file_system=True, ) -def test_create_archive_with_umask_should_call_attic_with_umask_parameters(): +def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): + insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1')) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + excludes_filename='excludes', + verbosity=None, + storage_config={}, + source_directories='foo bar', + repository='repo', + command='borg', + remote_path='borg1', + ) + + +def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) insert_platform_mock() insert_datetime_mock() @@ -174,12 +190,12 @@ def test_create_archive_with_umask_should_call_attic_with_umask_parameters(): storage_config={'umask': 740}, source_directories='foo bar', repository='repo', - command='attic', + command='borg', ) def test_create_archive_with_source_directories_glob_expands(): - insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food')) + insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) @@ -190,12 +206,12 @@ def test_create_archive_with_source_directories_glob_expands(): storage_config={}, source_directories='foo*', repository='repo', - command='attic', + command='borg', ) def test_create_archive_with_non_matching_source_directories_glob_passes_through(): - insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo*')) + insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*')) insert_platform_mock() insert_datetime_mock() flexmock(module).should_receive('glob').with_args('foo*').and_return([]) @@ -206,12 +222,12 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through storage_config={}, source_directories='foo*', repository='repo', - command='attic', + command='borg', ) -def test_create_archive_with_glob_should_call_attic_with_expanded_directories(): - insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food')) +def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): + insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) @@ -222,7 +238,7 @@ def test_create_archive_with_glob_should_call_attic_with_expanded_directories(): storage_config={}, source_directories='foo*', repository='repo', - command='attic', + command='borg', ) @@ -248,11 +264,11 @@ def test_make_prune_flags_should_return_flags_from_config(): PRUNE_COMMAND = ( - 'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', + 'borg', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', ) -def test_prune_archives_should_call_attic_with_parameters(): +def test_prune_archives_should_call_borg_with_parameters(): retention_config = flexmock() flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, @@ -263,11 +279,11 @@ def test_prune_archives_should_call_attic_with_parameters(): verbosity=None, repository='repo', retention_config=retention_config, - command='attic', + command='borg', ) -def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_parameter(): +def test_prune_archives_with_verbosity_some_should_call_borg_with_stats_parameter(): retention_config = flexmock() flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, @@ -278,11 +294,11 @@ def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_paramet repository='repo', verbosity=VERBOSITY_SOME, retention_config=retention_config, - command='attic', + command='borg', ) -def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter(): +def test_prune_archives_with_verbosity_lots_should_call_borg_with_verbose_parameter(): retention_config = flexmock() flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, @@ -293,7 +309,22 @@ def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_param repository='repo', verbosity=VERBOSITY_LOTS, retention_config=retention_config, - command='attic', + command='borg', + ) + +def test_prune_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): + retention_config = flexmock() + flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( + BASE_PRUNE_FLAGS, + ) + insert_subprocess_mock(PRUNE_COMMAND + ('--remote-path', 'borg1')) + + module.prune_archives( + verbosity=None, + repository='repo', + retention_config=retention_config, + command='borg', + remote_path='borg1', ) @@ -345,7 +376,7 @@ def test_make_check_flags_with_last_returns_last_flag(): assert flags == ('--last', 3) -def test_check_archives_should_call_attic_with_parameters(): +def test_check_archives_should_call_borg_with_parameters(): checks = flexmock() check_last = flexmock() consistency_config = flexmock().should_receive('get').and_return(check_last).mock @@ -353,7 +384,7 @@ def test_check_archives_should_call_attic_with_parameters(): flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) stdout = flexmock() insert_subprocess_mock( - ('attic', 'check', 'repo'), + ('borg', 'check', 'repo'), stdout=stdout, stderr=STDOUT, ) insert_platform_mock() @@ -365,16 +396,16 @@ def test_check_archives_should_call_attic_with_parameters(): verbosity=None, repository='repo', consistency_config=consistency_config, - command='attic', + command='borg', ) -def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter(): +def test_check_archives_with_verbosity_some_should_call_borg_with_verbose_parameter(): consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( - ('attic', 'check', 'repo', '--verbose'), + ('borg', 'check', 'repo', '--verbose'), stdout=None, stderr=STDOUT, ) insert_platform_mock() @@ -384,16 +415,16 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param verbosity=VERBOSITY_SOME, repository='repo', consistency_config=consistency_config, - command='attic', + command='borg', ) -def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter(): +def test_check_archives_with_verbosity_lots_should_call_borg_with_verbose_parameter(): consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( - ('attic', 'check', 'repo', '--verbose'), + ('borg', 'check', 'repo', '--verbose'), stdout=None, stderr=STDOUT, ) insert_platform_mock() @@ -403,7 +434,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param verbosity=VERBOSITY_LOTS, repository='repo', consistency_config=consistency_config, - command='attic', + command='borg', ) @@ -416,5 +447,30 @@ def test_check_archives_without_any_checks_should_bail(): verbosity=None, repository='repo', consistency_config=consistency_config, - command='attic', + command='borg', + ) + + +def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters(): + checks = flexmock() + check_last = flexmock() + consistency_config = flexmock().should_receive('get').and_return(check_last).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) + stdout = flexmock() + insert_subprocess_mock( + ('borg', 'check', 'repo', '--remote-path', 'borg1'), + stdout=stdout, stderr=STDOUT, + ) + insert_platform_mock() + insert_datetime_mock() + builtins_mock().should_receive('open').and_return(stdout) + flexmock(module.os).should_receive('devnull') + + module.check_archives( + verbosity=None, + repository='repo', + consistency_config=consistency_config, + command='borg', + remote_path='borg1', ) diff --git a/sample/config b/sample/config index acea8d7..f081e4c 100644 --- a/sample/config +++ b/sample/config @@ -3,10 +3,12 @@ # Globs are expanded. source_directories: /home /etc /var/log/syslog* -# For Borg only, you can specify to stay in same file system (do not cross -# mount points). +# Stay in same file system (do not cross mount points). #one_file_system: True +# Alternate Borg remote executable (defaults to "borg"): +#remote_path: borg1 + # Path to local or remote repository. repository: user@backupserver:sourcehostname.borg @@ -14,10 +16,12 @@ repository: user@backupserver:sourcehostname.borg # Passphrase to unlock the encryption key with. Only use on repositories that # were initialized with passphrase/repokey encryption. #encryption_passphrase: foo + # Type of compression to use when creating archives. See # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create # for details. Defaults to no compression. #compression: lz4 + # Umask to be used for borg create. #umask: 0740 @@ -30,6 +34,7 @@ keep_daily: 7 keep_weekly: 4 keep_monthly: 6 keep_yearly: 1 + #prefix: sourcehostname [consistency] @@ -38,5 +43,6 @@ keep_yearly: 1 # checks. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check # for details. checks: repository archives + # Restrict the number of checked archives to the last n. #check_last: 3 diff --git a/setup.py b/setup.py index 19c1c60..91d6f38 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.0.0' +VERSION = '1.0.1' setup( From 2e3e68d2cb955532f99251f53b79230350f1d5e9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 10 Jun 2016 13:34:23 -0700 Subject: [PATCH 108/189] Added tag 1.0.1 for changeset de2d7721cdec --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 84d6b8e..58dc3e1 100644 --- a/.hgtags +++ b/.hgtags @@ -27,3 +27,4 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4 28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8 0e1fbee9358de4f062fa9539e1355db83db70caa 1.0.0 +de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1 From 600c4389517a533691ed1cf3a11b9af517ad8e4d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 10 Jun 2016 17:11:28 -0700 Subject: [PATCH 109/189] Reverting to pre-rename issues link, because that link isn't yet renamed to borgmatic. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b9e1d93..c8bd5d9 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ backups. ## Issues and feedback Got an issue or an idea for a feature enhancement? Check out the [borgmatic -issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues). In +issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues). In order to create a new issue or comment on an issue, you'll need to [login first](https://tree.taiga.io/login). From 481dbc14c31ed0f971ffcb53b96b0abfc7582b31 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 12 Jun 2016 22:37:42 -0700 Subject: [PATCH 110/189] Rename issues URL. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c8bd5d9..b9e1d93 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ backups. ## Issues and feedback Got an issue or an idea for a feature enhancement? Check out the [borgmatic -issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues). In +issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues). In order to create a new issue or comment on an issue, you'll need to [login first](https://tree.taiga.io/login). From 938392b25b31d27230333a180e9e7a8b32e7b65c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 12 Jun 2016 22:40:04 -0700 Subject: [PATCH 111/189] Restricting issues list to open issues. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b9e1d93..1bae8c1 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ backups. ## Issues and feedback Got an issue or an idea for a feature enhancement? Check out the [borgmatic -issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues). In +issue tracker](https://tree.taiga.io/project/witten-borgmatic/issues?page=1&status=399951,399952,399955). In order to create a new issue or comment on an issue, you'll need to [login first](https://tree.taiga.io/login). From b22b552bf3f11dc2f99c36b4539000e22331c7cf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 13 Jun 2016 08:53:41 -0700 Subject: [PATCH 112/189] #20: Fix for traceback when remote_path option is missing. --- NEWS | 4 ++++ borgmatic/command.py | 2 +- setup.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index af155a9..25d95fd 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.0.2 + + * #20: Fix for traceback when remote_path option is missing. + 1.0.1 * #19: Support for Borg's --remote-path option to use an alternate Borg diff --git a/borgmatic/command.py b/borgmatic/command.py index bf6713d..784b74a 100644 --- a/borgmatic/command.py +++ b/borgmatic/command.py @@ -45,7 +45,7 @@ def main(): args = parse_arguments(*sys.argv[1:]) config = parse_configuration(args.config_filename, CONFIG_FORMAT) repository = config.location['repository'] - remote_path = config.location['remote_path'] + remote_path = config.location.get('remote_path') borg.initialize(config.storage) borg.create_archive( diff --git a/setup.py b/setup.py index 91d6f38..55fa95e 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.0.1' +VERSION = '1.0.2' setup( From ead991dcd18eb30934ee09aa42ffcae1057037a8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 13 Jun 2016 12:02:37 -0700 Subject: [PATCH 113/189] Added tag 1.0.2 for changeset 9603d13910b3 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 58dc3e1..02c0be7 100644 --- a/.hgtags +++ b/.hgtags @@ -28,3 +28,4 @@ e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4 dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8 0e1fbee9358de4f062fa9539e1355db83db70caa 1.0.0 de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1 +9603d13910b32d57a887765cab694ac5d0acc1f4 1.0.2 From 6bfe524bac6d0a1fd4b3091da964af900eb2d2cb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 23 Jun 2016 07:13:25 -0700 Subject: [PATCH 114/189] #21: Fix for verbosity flag not actually causing verbose output. --- NEWS | 4 ++++ borgmatic/borg.py | 12 ++++++------ borgmatic/tests/unit/test_borg.py | 24 ++++++++++++------------ setup.py | 2 +- tox.ini | 2 +- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/NEWS b/NEWS index 25d95fd..2cb9b09 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.0.3 + + * #21: Fix for verbosity flag not actually causing verbose output. + 1.0.2 * #20: Fix for traceback when remote_path option is missing. diff --git a/borgmatic/borg.py b/borgmatic/borg.py index 27c570a..f4c5ba9 100644 --- a/borgmatic/borg.py +++ b/borgmatic/borg.py @@ -41,8 +41,8 @@ def create_archive( one_file_system_flags = ('--one-file-system',) if one_file_system else () remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { - VERBOSITY_SOME: ('--stats',), - VERBOSITY_LOTS: ('--verbose', '--stats'), + VERBOSITY_SOME: ('--info', '--stats',), + VERBOSITY_LOTS: ('--debug', '--list', '--stats'), }.get(verbosity, ()) full_command = ( @@ -88,8 +88,8 @@ def prune_archives(verbosity, repository, retention_config, command=COMMAND, rem ''' remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { - VERBOSITY_SOME: ('--stats',), - VERBOSITY_LOTS: ('--verbose', '--stats'), + VERBOSITY_SOME: ('--info', '--stats',), + VERBOSITY_LOTS: ('--debug', '--stats'), }.get(verbosity, ()) full_command = ( @@ -171,8 +171,8 @@ def check_archives(verbosity, repository, consistency_config, command=COMMAND, r remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { - VERBOSITY_SOME: ('--verbose',), - VERBOSITY_LOTS: ('--verbose',), + VERBOSITY_SOME: ('--info',), + VERBOSITY_LOTS: ('--debug',), }.get(verbosity, ()) full_command = ( diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 0511357..1d146ba 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -102,8 +102,8 @@ def test_create_archive_with_none_excludes_filename_should_call_borg_without_exc ) -def test_create_archive_with_verbosity_some_should_call_borg_with_stats_parameter(): - insert_subprocess_mock(CREATE_COMMAND + ('--stats',)) +def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter(): + insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',)) insert_platform_mock() insert_datetime_mock() @@ -117,8 +117,8 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_stats_paramete ) -def test_create_archive_with_verbosity_lots_should_call_borg_with_verbose_parameter(): - insert_subprocess_mock(CREATE_COMMAND + ('--verbose', '--stats')) +def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter(): + insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats')) insert_platform_mock() insert_datetime_mock() @@ -283,12 +283,12 @@ def test_prune_archives_should_call_borg_with_parameters(): ) -def test_prune_archives_with_verbosity_some_should_call_borg_with_stats_parameter(): +def test_prune_archives_with_verbosity_some_should_call_borg_with_info_parameter(): retention_config = flexmock() flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) - insert_subprocess_mock(PRUNE_COMMAND + ('--stats',)) + insert_subprocess_mock(PRUNE_COMMAND + ('--info', '--stats',)) module.prune_archives( repository='repo', @@ -298,12 +298,12 @@ def test_prune_archives_with_verbosity_some_should_call_borg_with_stats_paramete ) -def test_prune_archives_with_verbosity_lots_should_call_borg_with_verbose_parameter(): +def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_parameter(): retention_config = flexmock() flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) - insert_subprocess_mock(PRUNE_COMMAND + ('--verbose', '--stats',)) + insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats',)) module.prune_archives( repository='repo', @@ -400,12 +400,12 @@ def test_check_archives_should_call_borg_with_parameters(): ) -def test_check_archives_with_verbosity_some_should_call_borg_with_verbose_parameter(): +def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter(): consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( - ('borg', 'check', 'repo', '--verbose'), + ('borg', 'check', 'repo', '--info'), stdout=None, stderr=STDOUT, ) insert_platform_mock() @@ -419,12 +419,12 @@ def test_check_archives_with_verbosity_some_should_call_borg_with_verbose_parame ) -def test_check_archives_with_verbosity_lots_should_call_borg_with_verbose_parameter(): +def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter(): consistency_config = flexmock().should_receive('get').and_return(None).mock flexmock(module).should_receive('_parse_checks').and_return(flexmock()) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( - ('borg', 'check', 'repo', '--verbose'), + ('borg', 'check', 'repo', '--debug'), stdout=None, stderr=STDOUT, ) insert_platform_mock() diff --git a/setup.py b/setup.py index 55fa95e..d4dbbbe 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.0.2' +VERSION = '1.0.3' setup( diff --git a/tox.ini b/tox.ini index 64d1f6c..90c5ce2 100644 --- a/tox.ini +++ b/tox.ini @@ -5,4 +5,4 @@ skipsdist=True [testenv] usedevelop=True deps=-rtest_requirements.txt -commands = py.test [] +commands = py.test borgmatic [] From f6d2e983d989a89308654547b9ffde7eb004d0d2 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 23 Jun 2016 07:13:29 -0700 Subject: [PATCH 115/189] Added tag 1.0.3 for changeset 32c6341dda9f --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 02c0be7..06c448a 100644 --- a/.hgtags +++ b/.hgtags @@ -29,3 +29,4 @@ dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8 0e1fbee9358de4f062fa9539e1355db83db70caa 1.0.0 de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1 9603d13910b32d57a887765cab694ac5d0acc1f4 1.0.2 +32c6341dda9fad77a3982641bce8a3a45821842e 1.0.3 From 5bd1cc55804ae333dc264942b11fa58b506a9ef6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 3 Jul 2016 22:07:53 -0700 Subject: [PATCH 116/189] #18: Fix for README mention of sample files not included in package. Also, added logo. --- NEWS | 5 +++++ README.md | 24 ++++++++++++++++++++---- setup.py | 2 +- static/borgmatic.svg | 1 + 4 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 static/borgmatic.svg diff --git a/NEWS b/NEWS index 2cb9b09..724a7c7 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.0.3-dev + + * #18: Fix for README mention of sample files not included in package. + * Added logo. + 1.0.3 * #21: Fix for verbosity flag not actually causing verbose output. diff --git a/README.md b/README.md index 1bae8c1..00c2b97 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -title: Borgmatic +title: borgmatic + +borgmatic logo ## Overview @@ -57,11 +59,14 @@ To install borgmatic, run the following command to download and install it: sudo pip install --upgrade borgmatic -Then, copy the following configuration files: +Then, download a [sample config +file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/config) and a +[sample excludes +file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/excludes). From the +directory where you downloaded them: - sudo cp sample/borgmatic.cron /etc/cron.d/borgmatic sudo mkdir /etc/borgmatic/ - sudo cp sample/config sample/excludes /etc/borgmatic/ + sudo mv config excludes /etc/borgmatic/ Lastly, modify the /etc files with your desired configuration. @@ -111,6 +116,17 @@ If you'd like to see the available command-line arguments, view the help: borgmatic --help +## Autopilot + +If you want to run borgmatic automatically, say once a day, the you can +configure a job runner like cron to invoke it periodically. Download the +[sample cron +file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/borgmatic.cron). +Then, from the directory where you downloaded it: + + sudo cp borgmatic.cron /etc/cron.d/borgmatic + + ## Running tests First install tox, which is used for setting up testing environments: diff --git a/setup.py b/setup.py index d4dbbbe..63b0081 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.0.3' +VERSION = '1.0.3-dev' setup( diff --git a/static/borgmatic.svg b/static/borgmatic.svg new file mode 100644 index 0000000..1c30816 --- /dev/null +++ b/static/borgmatic.svg @@ -0,0 +1 @@ + \ No newline at end of file From abb6bed4598206f4faa3abc5f7d1605b2ba4c4b4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 4 Jul 2016 09:19:34 -0700 Subject: [PATCH 117/189] Sample files for triggering borgmatic from a systemd timer. --- NEWS | 1 + README.md | 29 +++++++++++++++++++---- sample/{borgmatic.cron => cron/borgmatic} | 0 sample/systemd/borgmatic.service | 6 +++++ sample/systemd/borgmatic.timer | 8 +++++++ 5 files changed, 40 insertions(+), 4 deletions(-) rename sample/{borgmatic.cron => cron/borgmatic} (100%) create mode 100644 sample/systemd/borgmatic.service create mode 100644 sample/systemd/borgmatic.timer diff --git a/NEWS b/NEWS index 724a7c7..3dc98c5 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 1.0.3-dev * #18: Fix for README mention of sample files not included in package. + * #22: Sample files for triggering borgmatic from a systemd timer. * Added logo. 1.0.3 diff --git a/README.md b/README.md index 00c2b97..4d7f18e 100644 --- a/README.md +++ b/README.md @@ -119,12 +119,33 @@ If you'd like to see the available command-line arguments, view the help: ## Autopilot If you want to run borgmatic automatically, say once a day, the you can -configure a job runner like cron to invoke it periodically. Download the -[sample cron -file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/borgmatic.cron). +configure a job runner to invoke it periodically. + +### cron + +If you're using cron, download the [sample cron +file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/cron/borgmatic). Then, from the directory where you downloaded it: - sudo cp borgmatic.cron /etc/cron.d/borgmatic + sudo mv borgmatic /etc/cron.d/borgmatic + +You can modify the cron file if you'd like to run borgmatic more or less frequently. + +### systemd + +If you're using systemd instead of cron to run jobs, download the [sample +systemd service +file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/systemd/borgmatic.service) +and the [sample systemd timer +file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/systemd/borgmatic.timer). +Then, from the directory where you downloaded them: + + sudo mv {borgmatic.service,borgmatic.timer} /etc/systemd/system/ + sudo systemctl enable borgmatic.timer + sudo systemctl start borgmatic.timer + +Feel free to modify the timer file based on how frequently you'd like +borgmatic to run. ## Running tests diff --git a/sample/borgmatic.cron b/sample/cron/borgmatic similarity index 100% rename from sample/borgmatic.cron rename to sample/cron/borgmatic diff --git a/sample/systemd/borgmatic.service b/sample/systemd/borgmatic.service new file mode 100644 index 0000000..443b1f0 --- /dev/null +++ b/sample/systemd/borgmatic.service @@ -0,0 +1,6 @@ +[Unit] +Description=borgmatic backup + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/borgmatic diff --git a/sample/systemd/borgmatic.timer b/sample/systemd/borgmatic.timer new file mode 100644 index 0000000..4b49d1f --- /dev/null +++ b/sample/systemd/borgmatic.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Run borgmatic backup + +[Timer] +OnCalendar=daily + +[Install] +WantedBy=timers.target From 87c65fb7239706920523162391c01aa3c9d43e6b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 4 Jul 2016 09:35:51 -0700 Subject: [PATCH 118/189] Removing unnecessary curlies from bash command. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d7f18e..a8a0a03 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ and the [sample systemd timer file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/systemd/borgmatic.timer). Then, from the directory where you downloaded them: - sudo mv {borgmatic.service,borgmatic.timer} /etc/systemd/system/ + sudo mv borgmatic.service borgmatic.timer /etc/systemd/system/ sudo systemctl enable borgmatic.timer sudo systemctl start borgmatic.timer From 1aaf27dfb218c155b9304247ee94d1435dd99f11 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 25 Jun 2017 10:36:36 -0700 Subject: [PATCH 119/189] Changed example umask config to be more realistic. --- sample/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample/config b/sample/config index f081e4c..e778aaa 100644 --- a/sample/config +++ b/sample/config @@ -23,7 +23,7 @@ repository: user@backupserver:sourcehostname.borg #compression: lz4 # Umask to be used for borg create. -#umask: 0740 +#umask: 0077 [retention] # Retention policy for how many backups to keep in each category. See From e00f74ddf704201e8c110b6f9681194f442561c5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 2 Jul 2017 17:18:33 -0700 Subject: [PATCH 120/189] Dropped Python 2 support. Now Python 3 only. --- NEWS | 1 + README.md | 3 +++ borgmatic/config.py | 8 +------- borgmatic/tests/builtins.py | 7 +------ borgmatic/tests/integration/test_config.py | 7 +------ tox.ini | 2 +- 6 files changed, 8 insertions(+), 20 deletions(-) diff --git a/NEWS b/NEWS index 3dc98c5..9124360 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ * #18: Fix for README mention of sample files not included in package. * #22: Sample files for triggering borgmatic from a systemd timer. + * Dropped Python 2 support. Now Python 3 only. * Added logo. 1.0.3 diff --git a/README.md b/README.md index a8a0a03..7eb0eb1 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,9 @@ To install borgmatic, run the following command to download and install it: sudo pip install --upgrade borgmatic +Make sure you're using Python 3, as borgmatic does not support Python 2. (You +may have to use "pip3" instead of "pip".) + Then, download a [sample config file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/config) and a [sample excludes diff --git a/borgmatic/config.py b/borgmatic/config.py index 36cb4f4..9c455b3 100644 --- a/borgmatic/config.py +++ b/borgmatic/config.py @@ -1,11 +1,5 @@ from collections import OrderedDict, namedtuple - -try: - # Python 2 - from ConfigParser import RawConfigParser -except ImportError: - # Python 3 - from configparser import RawConfigParser +from configparser import RawConfigParser Section_format = namedtuple('Section_format', ('name', 'options')) diff --git a/borgmatic/tests/builtins.py b/borgmatic/tests/builtins.py index ff48f93..53970cd 100644 --- a/borgmatic/tests/builtins.py +++ b/borgmatic/tests/builtins.py @@ -3,9 +3,4 @@ import sys def builtins_mock(): - try: - # Python 2 - return flexmock(sys.modules['__builtin__']) - except KeyError: - # Python 3 - return flexmock(sys.modules['builtins']) + return flexmock(sys.modules['builtins']) diff --git a/borgmatic/tests/integration/test_config.py b/borgmatic/tests/integration/test_config.py index e849b4f..56cad6c 100644 --- a/borgmatic/tests/integration/test_config.py +++ b/borgmatic/tests/integration/test_config.py @@ -1,9 +1,4 @@ -try: - # Python 2 - from cStringIO import StringIO -except ImportError: - # Python 3 - from io import StringIO +from io import StringIO from collections import OrderedDict import string diff --git a/tox.ini b/tox.ini index 90c5ce2..579a919 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py27,py34 +envlist=py34 skipsdist=True [testenv] From 6e85940d6308ea7fe03457a8a00b7298ad57e0fb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 4 Jul 2017 16:52:24 -0700 Subject: [PATCH 121/189] Basic YAML configuration file parsing. --- .hgignore | 1 + MANIFEST.in | 1 + NEWS | 2 +- borgmatic/command.py | 2 +- borgmatic/config/__init__.py | 0 borgmatic/{config.py => config/legacy.py} | 0 borgmatic/config/schema.yaml | 48 +++++++++++ borgmatic/config/yaml.py | 84 +++++++++++++++++++ borgmatic/tests/builtins.py | 6 -- .../tests/integration/config/__init__.py | 0 .../{test_config.py => config/test_legacy.py} | 2 +- borgmatic/tests/unit/config/__init__.py | 0 .../{test_config.py => config/test_legacy.py} | 2 +- borgmatic/tests/unit/test_borg.py | 6 +- sample/config.yaml | 54 ++++++++++++ setup.py | 10 ++- 16 files changed, 203 insertions(+), 15 deletions(-) create mode 100644 MANIFEST.in create mode 100644 borgmatic/config/__init__.py rename borgmatic/{config.py => config/legacy.py} (100%) create mode 100644 borgmatic/config/schema.yaml create mode 100644 borgmatic/config/yaml.py delete mode 100644 borgmatic/tests/builtins.py create mode 100644 borgmatic/tests/integration/config/__init__.py rename borgmatic/tests/integration/{test_config.py => config/test_legacy.py} (92%) create mode 100644 borgmatic/tests/unit/config/__init__.py rename borgmatic/tests/unit/{test_config.py => config/test_legacy.py} (99%) create mode 100644 sample/config.yaml diff --git a/.hgignore b/.hgignore index 4128076..f41ba93 100644 --- a/.hgignore +++ b/.hgignore @@ -2,6 +2,7 @@ syntax: glob *.egg-info *.pyc *.swp +.cache .tox build dist diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..20ff231 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include borgmatic/config/schema.yaml diff --git a/NEWS b/NEWS index 9124360..b5d4216 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.0.3-dev +1.1.0 * #18: Fix for README mention of sample files not included in package. * #22: Sample files for triggering borgmatic from a systemd timer. diff --git a/borgmatic/command.py b/borgmatic/command.py index 784b74a..ef7bbee 100644 --- a/borgmatic/command.py +++ b/borgmatic/command.py @@ -5,7 +5,7 @@ from subprocess import CalledProcessError import sys from borgmatic import borg -from borgmatic.config import parse_configuration, CONFIG_FORMAT +from borgmatic.config.legacy import parse_configuration, CONFIG_FORMAT DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config' diff --git a/borgmatic/config/__init__.py b/borgmatic/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/borgmatic/config.py b/borgmatic/config/legacy.py similarity index 100% rename from borgmatic/config.py rename to borgmatic/config/legacy.py diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml new file mode 100644 index 0000000..83c50bb --- /dev/null +++ b/borgmatic/config/schema.yaml @@ -0,0 +1,48 @@ +map: + location: + required: True + map: + source_directories: + required: True + seq: + - type: scalar + one_file_system: + type: bool + remote_path: + type: scalar + repository: + required: True + type: scalar + storage: + map: + encryption_passphrase: + type: scalar + compression: + type: scalar + umask: + type: scalar + retention: + map: + keep_within: + type: scalar + keep_hourly: + type: int + keep_daily: + type: int + keep_weekly: + type: int + keep_monthly: + type: int + keep_yearly: + type: int + prefix: + type: scalar + consistency: + map: + checks: + seq: + - type: str + enum: ['repository', 'archives', 'disabled'] + unique: True + check_last: + type: int diff --git a/borgmatic/config/yaml.py b/borgmatic/config/yaml.py new file mode 100644 index 0000000..4ff01b9 --- /dev/null +++ b/borgmatic/config/yaml.py @@ -0,0 +1,84 @@ +import logging +import sys +import warnings + +import pkg_resources +import pykwalify.core +import pykwalify.errors +import ruamel.yaml.error + + +def schema_filename(): + ''' + Path to the installed YAML configuration schema file, used to validate and parse the + configuration. + ''' + return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml') + + +class Validation_error(ValueError): + ''' + A collection of error message strings generated when attempting to validate a particular + configurartion file. + ''' + def __init__(self, config_filename, error_messages): + self.config_filename = config_filename + self.error_messages = error_messages + + +def parse_configuration(config_filename, schema_filename): + ''' + Given the path to a config filename in YAML format and the path to a schema filename in + pykwalify YAML schema format, return the parsed configuration as a data structure of nested + dicts and lists corresponding to the schema. Example return value: + + {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'}, + 'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}} + + Raise FileNotFoundError if the file does not exist, PermissionError if the user does not + have permissions to read the file, or Validation_error if the config does not match the schema. + ''' + warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning) + logging.getLogger('pykwalify').setLevel(logging.CRITICAL) + + try: + validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename]) + except pykwalify.errors.CoreError as error: + if 'do not exists on disk' in str(error): + raise FileNotFoundError("No such file or directory: '{}'".format(config_filename)) + if 'Unable to load any data' in str(error): + # If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful. + # So reach back to the originating exception from ruamel.yaml for something more useful. + raise Validation_error(config_filename, (error.__context__,)) + raise + + parsed_result = validator.validate(raise_exception=False) + + if validator.validation_errors: + raise Validation_error(config_filename, validator.validation_errors) + + return parsed_result + + +def display_validation_error(validation_error): + ''' + Given a Validation_error, display its error messages to stderr. + ''' + print( + 'An error occurred while parsing a configuration file at {}:'.format( + validation_error.config_filename + ), + file=sys.stderr, + ) + + for error in validation_error.error_messages: + print(error, file=sys.stderr) + + +# FOR TESTING +if __name__ == '__main__': + try: + configuration = parse_configuration('sample/config.yaml', schema_filename()) + print(configuration) + except Validation_error as error: + display_validation_error(error) diff --git a/borgmatic/tests/builtins.py b/borgmatic/tests/builtins.py deleted file mode 100644 index 53970cd..0000000 --- a/borgmatic/tests/builtins.py +++ /dev/null @@ -1,6 +0,0 @@ -from flexmock import flexmock -import sys - - -def builtins_mock(): - return flexmock(sys.modules['builtins']) diff --git a/borgmatic/tests/integration/config/__init__.py b/borgmatic/tests/integration/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/borgmatic/tests/integration/test_config.py b/borgmatic/tests/integration/config/test_legacy.py similarity index 92% rename from borgmatic/tests/integration/test_config.py rename to borgmatic/tests/integration/config/test_legacy.py index 56cad6c..13ee654 100644 --- a/borgmatic/tests/integration/test_config.py +++ b/borgmatic/tests/integration/config/test_legacy.py @@ -3,7 +3,7 @@ from io import StringIO from collections import OrderedDict import string -from borgmatic import config as module +from borgmatic.config import legacy as module def test_parse_section_options_with_punctuation_should_return_section_options(): diff --git a/borgmatic/tests/unit/config/__init__.py b/borgmatic/tests/unit/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/borgmatic/tests/unit/test_config.py b/borgmatic/tests/unit/config/test_legacy.py similarity index 99% rename from borgmatic/tests/unit/test_config.py rename to borgmatic/tests/unit/config/test_legacy.py index 01e21bb..9e95cac 100644 --- a/borgmatic/tests/unit/test_config.py +++ b/borgmatic/tests/unit/config/test_legacy.py @@ -3,7 +3,7 @@ from collections import OrderedDict from flexmock import flexmock import pytest -from borgmatic import config as module +from borgmatic.config import legacy as module def test_option_should_create_config_option(): diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 1d146ba..3c2aa1c 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -1,11 +1,11 @@ from collections import OrderedDict from subprocess import STDOUT +import sys import os from flexmock import flexmock from borgmatic import borg as module -from borgmatic.tests.builtins import builtins_mock from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS @@ -389,7 +389,7 @@ def test_check_archives_should_call_borg_with_parameters(): ) insert_platform_mock() insert_datetime_mock() - builtins_mock().should_receive('open').and_return(stdout) + flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout) flexmock(module.os).should_receive('devnull') module.check_archives( @@ -464,7 +464,7 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param ) insert_platform_mock() insert_datetime_mock() - builtins_mock().should_receive('open').and_return(stdout) + flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout) flexmock(module.os).should_receive('devnull') module.check_archives( diff --git a/sample/config.yaml b/sample/config.yaml new file mode 100644 index 0000000..6ca27c7 --- /dev/null +++ b/sample/config.yaml @@ -0,0 +1,54 @@ +location: + # List of source directories to backup. Globs are expanded. + source_directories: + - /home + - /etc + - /var/log/syslog* + + # Stay in same file system (do not cross mount points). + #one_file_system: yes + + # Alternate Borg remote executable (defaults to "borg"): + #remote_path: borg1 + + # Path to local or remote repository. + repository: user@backupserver:sourcehostname.borg + +#storage: + # Passphrase to unlock the encryption key with. Only use on repositories + # that were initialized with passphrase/repokey encryption. + #encryption_passphrase: foo + + # Type of compression to use when creating archives. See + # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create + # for details. Defaults to no compression. + #compression: lz4 + + # Umask to be used for borg create. + #umask: 0077 + +retention: + # Retention policy for how many backups to keep in each category. See + # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for + # details. + #keep_within: 3H + #keep_hourly: 24 + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 6 + keep_yearly: 1 + + # When pruning, only consider archive names starting with this prefix. + #prefix: sourcehostname + +consistency: + # List of consistency checks to run: "repository", "archives", or both. + # Defaults to both. Set to "disabled" to disable all consistency checks. + # See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check + # for details. + checks: + - repository + - archives + + # Restrict the number of checked archives to the last n. + #check_last: 3 diff --git a/setup.py b/setup.py index 63b0081..198a2d5 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.0.3-dev' +VERSION = '1.1.0' setup( @@ -30,8 +30,14 @@ setup( obsoletes=[ 'atticmatic', ], + install_requires=( + 'pykwalify', + 'ruamel.yaml<=0.15', + 'setuptools', + ), tests_require=( 'flexmock', 'pytest', - ) + ), + include_package_data=True, ) From 1dc60d2856445564b2508db24e2e2399e7861203 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 4 Jul 2017 18:23:59 -0700 Subject: [PATCH 122/189] Integrating YAML config into borgmatic and updating README. --- README.md | 38 +++++++++++++++++++++----------------- borgmatic/command.py | 8 ++++---- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 7eb0eb1..5e69283 100644 --- a/README.md +++ b/README.md @@ -13,24 +13,28 @@ all on the command-line, and handles common errors. Here's an example config file: -```INI -[location] -# Space-separated list of source directories to backup. -# Globs are expanded. -source_directories: /home /etc /var/log/syslog* +```yaml +location: + # List of source directories to backup. Globs are expanded. + source_directories: + - /home + - /etc + - /var/log/syslog* -# Path to local or remote backup repository. -repository: user@backupserver:sourcehostname.borg + # Path to local or remote repository. + repository: user@backupserver:sourcehostname.borg -[retention] -# Retention policy for how many backups to keep in each category. -keep_daily: 7 -keep_weekly: 4 -keep_monthly: 6 +retention: + # Retention policy for how many backups to keep in each category. + keep_daily: 7 + keep_weekly: 4 + keep_monthly: 6 -[consistency] -# Consistency checks to run, or "disabled" to prevent checks. -checks: repository archives +consistency: + # List of consistency checks to run: "repository", "archives", or both. + checks: + - repository + - archives ``` Additionally, exclude patterns can be specified in a separate excludes config @@ -63,13 +67,13 @@ Make sure you're using Python 3, as borgmatic does not support Python 2. (You may have to use "pip3" instead of "pip".) Then, download a [sample config -file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/config) and a +file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/config.yaml) and a [sample excludes file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/excludes). From the directory where you downloaded them: sudo mkdir /etc/borgmatic/ - sudo mv config excludes /etc/borgmatic/ + sudo mv config.yaml excludes /etc/borgmatic/ Lastly, modify the /etc files with your desired configuration. diff --git a/borgmatic/command.py b/borgmatic/command.py index ef7bbee..cba6ef3 100644 --- a/borgmatic/command.py +++ b/borgmatic/command.py @@ -5,10 +5,10 @@ from subprocess import CalledProcessError import sys from borgmatic import borg -from borgmatic.config.legacy import parse_configuration, CONFIG_FORMAT +from borgmatic.config.yaml import parse_configuration, schema_filename -DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config' +DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' @@ -43,7 +43,7 @@ def parse_arguments(*arguments): def main(): try: args = parse_arguments(*sys.argv[1:]) - config = parse_configuration(args.config_filename, CONFIG_FORMAT) + config = parse_configuration(args.config_filename, schema_filename()) repository = config.location['repository'] remote_path = config.location.get('remote_path') @@ -53,6 +53,6 @@ def main(): ) borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path) borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path) - except (ValueError, IOError, CalledProcessError) as error: + except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) From bff6980eeedb2548198202f39f4799f00c6f2045 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 4 Jul 2017 18:32:37 -0700 Subject: [PATCH 123/189] Tests for YAML config code. --- .../tests/integration/config/test_yaml.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 borgmatic/tests/integration/config/test_yaml.py diff --git a/borgmatic/tests/integration/config/test_yaml.py b/borgmatic/tests/integration/config/test_yaml.py new file mode 100644 index 0000000..ebd33a3 --- /dev/null +++ b/borgmatic/tests/integration/config/test_yaml.py @@ -0,0 +1,113 @@ +import io +import string +import sys +import os + +from flexmock import flexmock +import pytest + +from borgmatic.config import yaml as module + + +def test_schema_filename_returns_plausable_path(): + schema_path = module.schema_filename() + + assert schema_path.endswith('/schema.yaml') + + +def mock_config_and_schema(config_yaml): + ''' + Set up mocks for the config config YAML string and the default schema so that pykwalify consumes + them when parsing the configuration. This is a little brittle in that it's relying on pykwalify + to open() the respective files in a particular order. + ''' + config_stream = io.StringIO(config_yaml) + schema_stream = open(module.schema_filename()) + builtins = flexmock(sys.modules['builtins']).should_call('open').mock + builtins.should_receive('open').and_return(config_stream).and_return(schema_stream) + flexmock(os.path).should_receive('exists').and_return(True) + + +def test_parse_configuration_transforms_file_into_mapping(): + mock_config_and_schema( + ''' + location: + source_directories: + - /home + - /etc + + repository: hostname.borg + + retention: + keep_daily: 7 + + consistency: + checks: + - repository + - archives + ''' + ) + + result = module.parse_configuration('config.yaml', 'schema.yaml') + + assert result == { + 'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'}, + 'retention': {'keep_daily': 7}, + 'consistency': {'checks': ['repository', 'archives']}, + } + + +def test_parse_configuration_passes_through_quoted_punctuation(): + escaped_punctuation = string.punctuation.replace('\\', r'\\').replace('"', r'\"') + + mock_config_and_schema( + ''' + location: + source_directories: + - /home + + repository: "{}.borg" + '''.format(escaped_punctuation) + ) + + result = module.parse_configuration('config.yaml', 'schema.yaml') + + assert result == { + 'location': { + 'source_directories': ['/home'], + 'repository': '{}.borg'.format(string.punctuation), + }, + } + + +def test_parse_configuration_raises_for_missing_config_file(): + with pytest.raises(FileNotFoundError): + module.parse_configuration('config.yaml', 'schema.yaml') + + +def test_parse_configuration_raises_for_missing_schema_file(): + mock_config_and_schema('') + flexmock(os.path).should_receive('exists').with_args('schema.yaml').and_return(False) + + with pytest.raises(FileNotFoundError): + module.parse_configuration('config.yaml', 'schema.yaml') + + +def test_parse_configuration_raises_for_syntax_error(): + mock_config_and_schema('invalid = yaml') + + with pytest.raises(module.Validation_error): + module.parse_configuration('config.yaml', 'schema.yaml') + + +def test_parse_configuration_raises_for_validation_error(): + mock_config_and_schema( + ''' + location: + source_directories: yes + repository: hostname.borg + ''' + ) + + with pytest.raises(module.Validation_error): + module.parse_configuration('config.yaml', 'schema.yaml') From 745de200dfcc43555e3f38b4bd6c1d5671dfa9e5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 8 Jul 2017 22:33:51 -0700 Subject: [PATCH 124/189] Basic YAML generating / validating / converting to. --- NEWS | 5 +- borgmatic/commands/__init__.py | 0 .../{command.py => commands/borgmatic.py} | 7 +- borgmatic/commands/convert_config.py | 54 +++++++++++ borgmatic/config/convert.py | 41 +++++++++ borgmatic/config/generate.py | 90 +++++++++++++++++++ borgmatic/config/schema.yaml | 62 +++++++++++++ borgmatic/config/{yaml.py => validate.py} | 33 +++---- .../tests/integration/commands/__init__.py | 0 .../test_borgmatic.py} | 2 +- .../config/{test_yaml.py => test_validate.py} | 13 +-- borgmatic/tests/unit/config/test_convert.py | 44 +++++++++ sample/config.yaml | 6 +- setup.py | 5 +- test_requirements.txt | 3 + tox.ini | 2 +- 16 files changed, 327 insertions(+), 40 deletions(-) create mode 100644 borgmatic/commands/__init__.py rename borgmatic/{command.py => commands/borgmatic.py} (83%) create mode 100644 borgmatic/commands/convert_config.py create mode 100644 borgmatic/config/convert.py create mode 100644 borgmatic/config/generate.py rename borgmatic/config/{yaml.py => validate.py} (65%) create mode 100644 borgmatic/tests/integration/commands/__init__.py rename borgmatic/tests/integration/{test_command.py => commands/test_borgmatic.py} (97%) rename borgmatic/tests/integration/config/{test_yaml.py => test_validate.py} (88%) create mode 100644 borgmatic/tests/unit/config/test_convert.py diff --git a/NEWS b/NEWS index b5d4216..3d16ba5 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ -1.1.0 +1.1.0.dev0 + * Switched config file format to YAML. Run convert-borgmatic-config to upgrade. + * Dropped Python 2 support. Now Python 3 only. * #18: Fix for README mention of sample files not included in package. * #22: Sample files for triggering borgmatic from a systemd timer. - * Dropped Python 2 support. Now Python 3 only. * Added logo. 1.0.3 diff --git a/borgmatic/commands/__init__.py b/borgmatic/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/borgmatic/command.py b/borgmatic/commands/borgmatic.py similarity index 83% rename from borgmatic/command.py rename to borgmatic/commands/borgmatic.py index cba6ef3..c817976 100644 --- a/borgmatic/command.py +++ b/borgmatic/commands/borgmatic.py @@ -5,7 +5,7 @@ from subprocess import CalledProcessError import sys from borgmatic import borg -from borgmatic.config.yaml import parse_configuration, schema_filename +from borgmatic.config.validate import parse_configuration, schema_filename DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' @@ -14,9 +14,8 @@ DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' def parse_arguments(*arguments): ''' - Given the name of the command with which this script was invoked and command-line arguments, - parse the arguments and return them as an ArgumentParser instance. Use the command name to - determine the default configuration and excludes paths. + Given command-line arguments with which this script was invoked, parse the arguments and return + them as an ArgumentParser instance. ''' parser = ArgumentParser() parser.add_argument( diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py new file mode 100644 index 0000000..8f645c0 --- /dev/null +++ b/borgmatic/commands/convert_config.py @@ -0,0 +1,54 @@ +from __future__ import print_function +from argparse import ArgumentParser +import os +from subprocess import CalledProcessError +import sys + +from ruamel import yaml + +from borgmatic import borg +from borgmatic.config import convert, generate, legacy, validate + + +DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config' +DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' +DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' + + +def parse_arguments(*arguments): + ''' + Given command-line arguments with which this script was invoked, parse the arguments and return + them as an ArgumentParser instance. + ''' + parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.') + parser.add_argument( + '-s', '--source', + dest='source_filename', + default=DEFAULT_SOURCE_CONFIG_FILENAME, + help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME), + ) + parser.add_argument( + '-d', '--destination', + dest='destination_filename', + default=DEFAULT_DESTINATION_CONFIG_FILENAME, + help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME), + ) + + return parser.parse_args(arguments) + + +def main(): + try: + args = parse_arguments(*sys.argv[1:]) + source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT) + schema = yaml.round_trip_load(open(validate.schema_filename()).read()) + + destination_config = convert.convert_legacy_parsed_config(source_config, schema) + + generate.write_configuration(args.destination_filename, destination_config) + + # TODO: As a backstop, check that the written config can actually be read and parsed, and + # that it matches the destination config data structure that was written. + except (ValueError, OSError) as error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py new file mode 100644 index 0000000..72b9ba3 --- /dev/null +++ b/borgmatic/config/convert.py @@ -0,0 +1,41 @@ +from ruamel import yaml + +from borgmatic.config import generate + + +def _convert_section(source_section_config, section_schema): + ''' + Given a legacy Parsed_config instance for a single section, convert it to its corresponding + yaml.comments.CommentedMap representation in preparation for actual serialization to YAML. + + Additionally, use the section schema as a source of helpful comments to include within the + returned CommentedMap. + ''' + destination_section_config = yaml.comments.CommentedMap(source_section_config) + generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT) + + return destination_section_config + + +def convert_legacy_parsed_config(source_config, schema): + ''' + Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its + corresponding yaml.comments.CommentedMap representation in preparation for actual serialization + to YAML. + + Additionally, use the given schema as a source of helpful comments to include within the + returned CommentedMap. + ''' + destination_config = yaml.comments.CommentedMap([ + (section_name, _convert_section(section_config, schema['map'][section_name])) + for section_name, section_config in source_config._asdict().items() + ]) + + destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ') + + if source_config.consistency['checks']: + destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') + + generate.add_comments_to_configuration(destination_config, schema) + + return destination_config diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py new file mode 100644 index 0000000..cf4861b --- /dev/null +++ b/borgmatic/config/generate.py @@ -0,0 +1,90 @@ +from collections import OrderedDict + +from ruamel import yaml + + +INDENT = 4 + + +def write_configuration(config_filename, config): + ''' + Given a target config filename and a config data structure of nested OrderedDicts, write out the + config to file as YAML. + ''' + with open(config_filename, 'w') as config_file: + config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)) + + +def _insert_newline_before_comment(config, field_name): + ''' + Using some ruamel.yaml black magic, insert a blank line in the config right befor the given + field and its comments. + ''' + config.ca.items[field_name][1].insert( + 0, + yaml.tokens.CommentToken('\n', yaml.error.CommentMark(0), None), + ) + + +def add_comments_to_configuration(config, schema, indent=0): + ''' + Using descriptions from a schema as a source, add those descriptions as comments to the given + config before each field. This function only adds comments for the top-most config map level. + Indent the comment the given number of characters. + ''' + for index, field_name in enumerate(config.keys()): + field_schema = schema['map'].get(field_name, {}) + description = field_schema.get('desc') + + # No description to use? Skip it. + if not schema or not description: + continue + + config.yaml_set_comment_before_after_key( + key=field_name, + before=description, + indent=indent, + ) + if index > 0: + _insert_newline_before_comment(config, field_name) + + +def _section_schema_to_sample_configuration(section_schema): + ''' + Given the schema for a particular config section, generate and return sample config for that + section. Include comments for each field based on the schema "desc" description. + ''' + section_config = yaml.comments.CommentedMap([ + (field_name, field_schema['example']) + for field_name, field_schema in section_schema['map'].items() + ]) + + add_comments_to_configuration(section_config, section_schema, indent=INDENT) + + return section_config + + +def _schema_to_sample_configuration(schema): + ''' + Given a loaded configuration schema, generate and return sample config for it. Include comments + for each section based on the schema "desc" description. + ''' + config = yaml.comments.CommentedMap([ + (section_name, _section_schema_to_sample_configuration(section_schema)) + for section_name, section_schema in schema['map'].items() + ]) + + add_comments_to_configuration(config, schema) + + return config + + +def generate_sample_configuration(config_filename, schema_filename): + ''' + Given a target config filename and the path to a schema filename in pykwalify YAML schema + format, write out a sample configuration file based on that schema. + ''' + schema = yaml.round_trip_load(open(schema_filename)) + config = _schema_to_sample_configuration(schema) + + write_configuration(config_filename, config) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 83c50bb..2cc2f4e 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1,48 +1,110 @@ +name: Borgmatic configuration file schema map: location: + desc: | + Where to look for files to backup, and where to store those backups. See + https://borgbackup.readthedocs.io/en/stable/quickstart.html and + https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details. required: True map: source_directories: required: True seq: - type: scalar + desc: List of source directories to backup. Globs are expanded. + example: + - /home + - /etc + - /var/log/syslog* one_file_system: type: bool + desc: Stay in same file system (do not cross mount points). + example: yes remote_path: type: scalar + desc: Alternate Borg remote executable. Defaults to "borg". + example: borg1 repository: required: True type: scalar + desc: Path to local or remote repository. + example: user@backupserver:sourcehostname.borg storage: + desc: | + Repository storage options. See + https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and + https://borgbackup.readthedocs.io/en/stable/usage.html#environment-variables for details. map: encryption_passphrase: type: scalar + desc: | + Passphrase to unlock the encryption key with. Only use on repositories that were + initialized with passphrase/repokey encryption. Quote the value if it contains + punctuation, so it parses correctly. And backslash any quote or backslash + literals as well. + example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" compression: type: scalar + desc: | + Type of compression to use when creating archives. See + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details. + Defaults to no compression. + example: lz4 umask: type: scalar + desc: Umask to be used for borg create. + example: 0077 retention: + desc: | + Retention policy for how many backups to keep in each category. See + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details. map: keep_within: type: scalar + desc: Keep all archives within this time interval. + example: 3H keep_hourly: type: int + desc: Number of hourly archives to keep. + example: 24 keep_daily: type: int + desc: Number of daily archives to keep. + example: 7 keep_weekly: type: int + desc: Number of weekly archives to keep. + example: 4 keep_monthly: type: int + desc: Number of monthly archives to keep. + example: 6 keep_yearly: type: int + desc: Number of yearly archives to keep. + example: 1 prefix: type: scalar + desc: When pruning, only consider archive names starting with this prefix. + example: sourcehostname consistency: + desc: | + Consistency checks to run after backups. See + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. map: checks: seq: - type: str enum: ['repository', 'archives', 'disabled'] unique: True + desc: | + List of consistency checks to run: "repository", "archives", or both. Defaults + to both. Set to "disabled" to disable all consistency checks. See + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. + example: + - repository + - archives check_last: type: int + desc: Restrict the number of checked archives to the last n. + example: 3 diff --git a/borgmatic/config/yaml.py b/borgmatic/config/validate.py similarity index 65% rename from borgmatic/config/yaml.py rename to borgmatic/config/validate.py index 4ff01b9..5dae6f3 100644 --- a/borgmatic/config/yaml.py +++ b/borgmatic/config/validate.py @@ -5,7 +5,7 @@ import warnings import pkg_resources import pykwalify.core import pykwalify.errors -import ruamel.yaml.error +from ruamel import yaml def schema_filename(): @@ -38,20 +38,18 @@ def parse_configuration(config_filename, schema_filename): Raise FileNotFoundError if the file does not exist, PermissionError if the user does not have permissions to read the file, or Validation_error if the config does not match the schema. ''' - warnings.simplefilter('ignore', ruamel.yaml.error.UnsafeLoaderWarning) - logging.getLogger('pykwalify').setLevel(logging.CRITICAL) - try: - validator = pykwalify.core.Core(source_file=config_filename, schema_files=[schema_filename]) - except pykwalify.errors.CoreError as error: - if 'do not exists on disk' in str(error): - raise FileNotFoundError("No such file or directory: '{}'".format(config_filename)) - if 'Unable to load any data' in str(error): - # If the YAML file has a syntax error, pykwalify's exception is particularly unhelpful. - # So reach back to the originating exception from ruamel.yaml for something more useful. - raise Validation_error(config_filename, (error.__context__,)) - raise + schema = yaml.round_trip_load(open(schema_filename)) + except yaml.error.YAMLError as error: + raise Validation_error(config_filename, (str(error),)) + # pykwalify gets angry if the example field is not a string. So rather than bend to its will, + # simply remove all examples before passing the schema to pykwalify. + for section_name, section_schema in schema['map'].items(): + for field_name, field_schema in section_schema['map'].items(): + field_schema.pop('example') + + validator = pykwalify.core.Core(source_file=config_filename, schema_data=schema) parsed_result = validator.validate(raise_exception=False) if validator.validation_errors: @@ -73,12 +71,3 @@ def display_validation_error(validation_error): for error in validation_error.error_messages: print(error, file=sys.stderr) - - -# FOR TESTING -if __name__ == '__main__': - try: - configuration = parse_configuration('sample/config.yaml', schema_filename()) - print(configuration) - except Validation_error as error: - display_validation_error(error) diff --git a/borgmatic/tests/integration/commands/__init__.py b/borgmatic/tests/integration/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/borgmatic/tests/integration/test_command.py b/borgmatic/tests/integration/commands/test_borgmatic.py similarity index 97% rename from borgmatic/tests/integration/test_command.py rename to borgmatic/tests/integration/commands/test_borgmatic.py index c1e311c..69c2468 100644 --- a/borgmatic/tests/integration/test_command.py +++ b/borgmatic/tests/integration/commands/test_borgmatic.py @@ -4,7 +4,7 @@ import sys from flexmock import flexmock import pytest -from borgmatic import command as module +from borgmatic.commands import borgmatic as module def test_parse_arguments_with_no_arguments_uses_defaults(): diff --git a/borgmatic/tests/integration/config/test_yaml.py b/borgmatic/tests/integration/config/test_validate.py similarity index 88% rename from borgmatic/tests/integration/config/test_yaml.py rename to borgmatic/tests/integration/config/test_validate.py index ebd33a3..4769551 100644 --- a/borgmatic/tests/integration/config/test_yaml.py +++ b/borgmatic/tests/integration/config/test_validate.py @@ -6,7 +6,7 @@ import os from flexmock import flexmock import pytest -from borgmatic.config import yaml as module +from borgmatic.config import validate as module def test_schema_filename_returns_plausable_path(): @@ -18,13 +18,13 @@ def test_schema_filename_returns_plausable_path(): def mock_config_and_schema(config_yaml): ''' Set up mocks for the config config YAML string and the default schema so that pykwalify consumes - them when parsing the configuration. This is a little brittle in that it's relying on pykwalify - to open() the respective files in a particular order. + them when parsing the configuration. This is a little brittle in that it's relying on the code + under test to open() the respective files in a particular order. ''' - config_stream = io.StringIO(config_yaml) schema_stream = open(module.schema_filename()) + config_stream = io.StringIO(config_yaml) builtins = flexmock(sys.modules['builtins']).should_call('open').mock - builtins.should_receive('open').and_return(config_stream).and_return(schema_stream) + builtins.should_receive('open').and_return(schema_stream).and_return(config_stream) flexmock(os.path).should_receive('exists').and_return(True) @@ -87,7 +87,8 @@ def test_parse_configuration_raises_for_missing_config_file(): def test_parse_configuration_raises_for_missing_schema_file(): mock_config_and_schema('') - flexmock(os.path).should_receive('exists').with_args('schema.yaml').and_return(False) + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('schema.yaml').and_raise(FileNotFoundError) with pytest.raises(FileNotFoundError): module.parse_configuration('config.yaml', 'schema.yaml') diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py new file mode 100644 index 0000000..39fc385 --- /dev/null +++ b/borgmatic/tests/unit/config/test_convert.py @@ -0,0 +1,44 @@ +from collections import defaultdict, OrderedDict, namedtuple + +from borgmatic.config import convert as module + + +Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency')) + + +def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): + source_config = Parsed_config( + location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]), + storage=OrderedDict([('encryption_passphrase', 'supersecret')]), + retention=OrderedDict([('keep_daily', 7)]), + consistency=OrderedDict([('checks', 'repository')]), + ) + schema = {'map': defaultdict(lambda: {'map': {}})} + + destination_config = module.convert_legacy_parsed_config(source_config, schema) + + assert destination_config == OrderedDict([ + ('location', OrderedDict([('source_directories', ['/home']), ('repository', 'hostname.borg')])), + ('storage', OrderedDict([('encryption_passphrase', 'supersecret')])), + ('retention', OrderedDict([('keep_daily', 7)])), + ('consistency', OrderedDict([('checks', ['repository'])])), + ]) + + +def test_convert_legacy_parsed_config_splits_space_separated_values(): + source_config = Parsed_config( + location=OrderedDict([('source_directories', '/home /etc')]), + storage=OrderedDict(), + retention=OrderedDict(), + consistency=OrderedDict([('checks', 'repository archives')]), + ) + schema = {'map': defaultdict(lambda: {'map': {}})} + + destination_config = module.convert_legacy_parsed_config(source_config, schema) + + assert destination_config == OrderedDict([ + ('location', OrderedDict([('source_directories', ['/home', '/etc'])])), + ('storage', OrderedDict()), + ('retention', OrderedDict()), + ('consistency', OrderedDict([('checks', ['repository', 'archives'])])), + ]) diff --git a/sample/config.yaml b/sample/config.yaml index 6ca27c7..daac230 100644 --- a/sample/config.yaml +++ b/sample/config.yaml @@ -16,8 +16,10 @@ location: #storage: # Passphrase to unlock the encryption key with. Only use on repositories - # that were initialized with passphrase/repokey encryption. - #encryption_passphrase: foo + # that were initialized with passphrase/repokey encryption. Quote the value + # if it contains punctuation so it parses correctly. And backslash any + # quote or backslash literals as well. + #encryption_passphrase: "foo" # Type of compression to use when creating archives. See # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create diff --git a/setup.py b/setup.py index 198a2d5..8ffeaba 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.0' +VERSION = '1.1.0.dev0' setup( @@ -24,7 +24,8 @@ setup( packages=find_packages(), entry_points={ 'console_scripts': [ - 'borgmatic = borgmatic.command:main', + 'borgmatic = borgmatic.commands.borgmatic:main', + 'convert-borgmatic-config = borgmatic.commands.convert_config:main', ] }, obsoletes=[ diff --git a/test_requirements.txt b/test_requirements.txt index 3c5f9e5..6bf66e0 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,2 +1,5 @@ flexmock==0.10.2 +pykwalify==1.6.0 pytest==2.9.1 +pytest-cov==2.5.1 +ruamel.yaml==0.15.18 diff --git a/tox.ini b/tox.ini index 579a919..7f6a375 100644 --- a/tox.ini +++ b/tox.ini @@ -5,4 +5,4 @@ skipsdist=True [testenv] usedevelop=True deps=-rtest_requirements.txt -commands = py.test borgmatic [] +commands = py.test --cov=borgmatic borgmatic [] From e50fd04750cdc5175910b31b5da83713175cd00e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 8 Jul 2017 23:01:41 -0700 Subject: [PATCH 125/189] Adding test coverage report. Making tests a little less brittle. --- borgmatic/config/validate.py | 5 +++-- .../tests/integration/config/test_validate.py | 17 ++++++++--------- tox.ini | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 5dae6f3..82b4f93 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -39,6 +39,7 @@ def parse_configuration(config_filename, schema_filename): have permissions to read the file, or Validation_error if the config does not match the schema. ''' try: + config = yaml.round_trip_load(open(config_filename)) schema = yaml.round_trip_load(open(schema_filename)) except yaml.error.YAMLError as error: raise Validation_error(config_filename, (str(error),)) @@ -49,7 +50,7 @@ def parse_configuration(config_filename, schema_filename): for field_name, field_schema in section_schema['map'].items(): field_schema.pop('example') - validator = pykwalify.core.Core(source_file=config_filename, schema_data=schema) + validator = pykwalify.core.Core(source_data=config, schema_data=schema) parsed_result = validator.validate(raise_exception=False) if validator.validation_errors: @@ -58,7 +59,7 @@ def parse_configuration(config_filename, schema_filename): return parsed_result -def display_validation_error(validation_error): +def display_validation_error(validation_error): # pragma: no cover ''' Given a Validation_error, display its error messages to stderr. ''' diff --git a/borgmatic/tests/integration/config/test_validate.py b/borgmatic/tests/integration/config/test_validate.py index 4769551..897ea41 100644 --- a/borgmatic/tests/integration/config/test_validate.py +++ b/borgmatic/tests/integration/config/test_validate.py @@ -17,15 +17,14 @@ def test_schema_filename_returns_plausable_path(): def mock_config_and_schema(config_yaml): ''' - Set up mocks for the config config YAML string and the default schema so that pykwalify consumes - them when parsing the configuration. This is a little brittle in that it's relying on the code - under test to open() the respective files in a particular order. + Set up mocks for the config config YAML string and the default schema so that the code under + test consumes them when parsing the configuration. ''' - schema_stream = open(module.schema_filename()) config_stream = io.StringIO(config_yaml) - builtins = flexmock(sys.modules['builtins']).should_call('open').mock - builtins.should_receive('open').and_return(schema_stream).and_return(config_stream) - flexmock(os.path).should_receive('exists').and_return(True) + schema_stream = open(module.schema_filename()) + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('config.yaml').and_return(config_stream) + builtins.should_receive('open').with_args('schema.yaml').and_return(schema_stream) def test_parse_configuration_transforms_file_into_mapping(): @@ -95,9 +94,9 @@ def test_parse_configuration_raises_for_missing_schema_file(): def test_parse_configuration_raises_for_syntax_error(): - mock_config_and_schema('invalid = yaml') + mock_config_and_schema('foo:\nbar') - with pytest.raises(module.Validation_error): + with pytest.raises(ValueError): module.parse_configuration('config.yaml', 'schema.yaml') diff --git a/tox.ini b/tox.ini index 7f6a375..159847d 100644 --- a/tox.ini +++ b/tox.ini @@ -5,4 +5,4 @@ skipsdist=True [testenv] usedevelop=True deps=-rtest_requirements.txt -commands = py.test --cov=borgmatic borgmatic [] +commands = py.test --cov-report term-missing:skip-covered --cov=borgmatic borgmatic [] From a16d90ff464a11cd75bff89fac513d3acc647107 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Jul 2017 10:27:34 -0700 Subject: [PATCH 126/189] Adding a "does not raise" test for displaying errors. --- borgmatic/config/validate.py | 2 +- borgmatic/tests/integration/config/test_validate.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 82b4f93..3125176 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -59,7 +59,7 @@ def parse_configuration(config_filename, schema_filename): return parsed_result -def display_validation_error(validation_error): # pragma: no cover +def display_validation_error(validation_error): ''' Given a Validation_error, display its error messages to stderr. ''' diff --git a/borgmatic/tests/integration/config/test_validate.py b/borgmatic/tests/integration/config/test_validate.py index 897ea41..90e223d 100644 --- a/borgmatic/tests/integration/config/test_validate.py +++ b/borgmatic/tests/integration/config/test_validate.py @@ -111,3 +111,10 @@ def test_parse_configuration_raises_for_validation_error(): with pytest.raises(module.Validation_error): module.parse_configuration('config.yaml', 'schema.yaml') + + +def test_display_validation_error_does_not_raise(): + flexmock(sys.modules['builtins']).should_receive('print') + error = module.Validation_error('config.yaml', ('oops', 'uh oh')) + + module.display_validation_error(error) From 1bcb2a8be402508fcccad6925b14b4bd8da77c9f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Jul 2017 11:41:55 -0700 Subject: [PATCH 127/189] More test coverage, and simplification of config generation. --- .hgignore | 1 + borgmatic/commands/generate_config.py | 39 ++++++++++ borgmatic/config/generate.py | 72 +++++++++---------- .../tests/integration/config/test_generate.py | 43 +++++++++++ borgmatic/tests/unit/config/test_generate.py | 49 +++++++++++++ setup.py | 1 + 6 files changed, 165 insertions(+), 40 deletions(-) create mode 100644 borgmatic/commands/generate_config.py create mode 100644 borgmatic/tests/integration/config/test_generate.py create mode 100644 borgmatic/tests/unit/config/test_generate.py diff --git a/.hgignore b/.hgignore index f41ba93..fd7c96d 100644 --- a/.hgignore +++ b/.hgignore @@ -3,6 +3,7 @@ syntax: glob *.pyc *.swp .cache +.coverage .tox build dist diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py new file mode 100644 index 0000000..f326d6a --- /dev/null +++ b/borgmatic/commands/generate_config.py @@ -0,0 +1,39 @@ +from __future__ import print_function +from argparse import ArgumentParser +import os +from subprocess import CalledProcessError +import sys + +from ruamel import yaml + +from borgmatic import borg +from borgmatic.config import convert, generate, validate + + +DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' + + +def parse_arguments(*arguments): + ''' + Given command-line arguments with which this script was invoked, parse the arguments and return + them as an ArgumentParser instance. + ''' + parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.') + parser.add_argument( + '-d', '--destination', + dest='destination_filename', + default=DEFAULT_DESTINATION_CONFIG_FILENAME, + help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME), + ) + + return parser.parse_args(arguments) + + +def main(): + try: + args = parse_arguments(*sys.argv[1:]) + + generate.generate_sample_configuration(args.destination_filename, validate.schema_filename()) + except (ValueError, OSError) as error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index cf4861b..66d6c44 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -6,15 +6,6 @@ from ruamel import yaml INDENT = 4 -def write_configuration(config_filename, config): - ''' - Given a target config filename and a config data structure of nested OrderedDicts, write out the - config to file as YAML. - ''' - with open(config_filename, 'w') as config_file: - config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)) - - def _insert_newline_before_comment(config, field_name): ''' Using some ruamel.yaml black magic, insert a blank line in the config right befor the given @@ -26,6 +17,37 @@ def _insert_newline_before_comment(config, field_name): ) +def _schema_to_sample_configuration(schema, level=0): + ''' + Given a loaded configuration schema, generate and return sample config for it. Include comments + for each section based on the schema "desc" description. + ''' + example = schema.get('example') + if example: + return example + + config = yaml.comments.CommentedMap([ + ( + section_name, + _schema_to_sample_configuration(section_schema, level + 1), + ) + for section_name, section_schema in schema['map'].items() + ]) + + add_comments_to_configuration(config, schema, indent=(level * INDENT)) + + return config + + +def write_configuration(config_filename, config): + ''' + Given a target config filename and a config data structure of nested OrderedDicts, write out the + config to file as YAML. + ''' + with open(config_filename, 'w') as config_file: + config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)) + + def add_comments_to_configuration(config, schema, indent=0): ''' Using descriptions from a schema as a source, add those descriptions as comments to the given @@ -37,7 +59,7 @@ def add_comments_to_configuration(config, schema, indent=0): description = field_schema.get('desc') # No description to use? Skip it. - if not schema or not description: + if not field_schema or not description: continue config.yaml_set_comment_before_after_key( @@ -49,36 +71,6 @@ def add_comments_to_configuration(config, schema, indent=0): _insert_newline_before_comment(config, field_name) -def _section_schema_to_sample_configuration(section_schema): - ''' - Given the schema for a particular config section, generate and return sample config for that - section. Include comments for each field based on the schema "desc" description. - ''' - section_config = yaml.comments.CommentedMap([ - (field_name, field_schema['example']) - for field_name, field_schema in section_schema['map'].items() - ]) - - add_comments_to_configuration(section_config, section_schema, indent=INDENT) - - return section_config - - -def _schema_to_sample_configuration(schema): - ''' - Given a loaded configuration schema, generate and return sample config for it. Include comments - for each section based on the schema "desc" description. - ''' - config = yaml.comments.CommentedMap([ - (section_name, _section_schema_to_sample_configuration(section_schema)) - for section_name, section_schema in schema['map'].items() - ]) - - add_comments_to_configuration(config, schema) - - return config - - def generate_sample_configuration(config_filename, schema_filename): ''' Given a target config filename and the path to a schema filename in pykwalify YAML schema diff --git a/borgmatic/tests/integration/config/test_generate.py b/borgmatic/tests/integration/config/test_generate.py new file mode 100644 index 0000000..89d8467 --- /dev/null +++ b/borgmatic/tests/integration/config/test_generate.py @@ -0,0 +1,43 @@ +from io import StringIO +import sys + +from flexmock import flexmock + +from borgmatic.config import generate as module + + +def test_insert_newline_before_comment_does_not_raise(): + field_name = 'foo' + config = module.yaml.comments.CommentedMap([(field_name, 33)]) + config.yaml_set_comment_before_after_key(key=field_name, before='Comment',) + + module._insert_newline_before_comment(config, field_name) + + +def test_write_configuration_does_not_raise(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').and_return(StringIO()) + + module.write_configuration('config.yaml', {}) + + +def test_add_comments_to_configuration_does_not_raise(): + # Ensure that it can deal with fields both in the schema and missing from the schema. + config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)]) + schema = { + 'map': { + 'foo': {'desc': 'Foo'}, + 'bar': {'desc': 'Bar'}, + } + } + + module.add_comments_to_configuration(config, schema) + + +def test_generate_sample_configuration_does_not_raise(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('schema.yaml').and_return('') + flexmock(module).should_receive('write_configuration') + flexmock(module).should_receive('_schema_to_sample_configuration') + + module.generate_sample_configuration('config.yaml', 'schema.yaml') diff --git a/borgmatic/tests/unit/config/test_generate.py b/borgmatic/tests/unit/config/test_generate.py new file mode 100644 index 0000000..3536f23 --- /dev/null +++ b/borgmatic/tests/unit/config/test_generate.py @@ -0,0 +1,49 @@ +from collections import OrderedDict + +from flexmock import flexmock + +from borgmatic.config import generate as module + + +def test_schema_to_sample_configuration_generates_config_with_examples(): + flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) + flexmock(module).should_receive('add_comments_to_configuration') + schema = { + 'map': OrderedDict([ + ( + 'section1', { + 'map': { + 'field1': OrderedDict([ + ('example', 'Example 1') + ]), + }, + }, + ), + ( + 'section2', { + 'map': OrderedDict([ + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ]), + } + ), + ]) + } + + config = module._schema_to_sample_configuration(schema) + + assert config == OrderedDict([ + ( + 'section1', + OrderedDict([ + ('field1', 'Example 1'), + ]), + ), + ( + 'section2', + OrderedDict([ + ('field2', 'Example 2'), + ('field3', 'Example 3'), + ]), + ) + ]) diff --git a/setup.py b/setup.py index 8ffeaba..6e9d53a 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ setup( 'console_scripts': [ 'borgmatic = borgmatic.commands.borgmatic:main', 'convert-borgmatic-config = borgmatic.commands.convert_config:main', + 'generate-borgmatic-config = borgmatic.commands.generate_config:main', ] }, obsoletes=[ From dc9b075d5ab122f6527de533be6fc3277328bf6e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Jul 2017 11:48:24 -0700 Subject: [PATCH 128/189] Rename convert-borgmatic-config to upgrade-borgmatic-config. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 3d16ba5..3209b41 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ 1.1.0.dev0 - * Switched config file format to YAML. Run convert-borgmatic-config to upgrade. + * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade. * Dropped Python 2 support. Now Python 3 only. * #18: Fix for README mention of sample files not included in package. * #22: Sample files for triggering borgmatic from a systemd timer. diff --git a/setup.py b/setup.py index 6e9d53a..8f8da5c 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup( entry_points={ 'console_scripts': [ 'borgmatic = borgmatic.commands.borgmatic:main', - 'convert-borgmatic-config = borgmatic.commands.convert_config:main', + 'upgrade-borgmatic-config = borgmatic.commands.convert_config:main', 'generate-borgmatic-config = borgmatic.commands.generate_config:main', ] }, From d4ae7814a03d9caa612a84ae3eb6a6f378b18419 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Jul 2017 11:49:51 -0700 Subject: [PATCH 129/189] Adding TODO about a helpful notice about legacy config. --- borgmatic/commands/borgmatic.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index c817976..4c2e9ba 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -41,6 +41,9 @@ def parse_arguments(*arguments): def main(): try: + # TODO: Detect whether only legacy config is present. If so, inform the user about how to + # upgrade, then exet. + args = parse_arguments(*sys.argv[1:]) config = parse_configuration(args.config_filename, schema_filename()) repository = config.location['repository'] From d49be195448cae5e493113c8f851fe4ff1582c0f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Jul 2017 16:18:10 -0700 Subject: [PATCH 130/189] Add a version to the schema, because inevitably I'll want to revise the schema. --- borgmatic/config/schema.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 2cc2f4e..347dfee 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1,4 +1,5 @@ name: Borgmatic configuration file schema +version: 1 map: location: desc: | From 17c87f8758ce86be264964af9b69dd41538d1ff4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Jul 2017 17:03:45 -0700 Subject: [PATCH 131/189] Completed test coverage of commands (except for main()s). --- borgmatic/commands/borgmatic.py | 2 +- borgmatic/commands/convert_config.py | 3 ++- borgmatic/commands/generate_config.py | 2 +- .../integration/commands/test_convert_config.py | 14 ++++++++++++++ .../integration/commands/test_generate_config.py | 12 ++++++++++++ 5 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 borgmatic/tests/integration/commands/test_convert_config.py create mode 100644 borgmatic/tests/integration/commands/test_generate_config.py diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 4c2e9ba..1fcc582 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -39,7 +39,7 @@ def parse_arguments(*arguments): return parser.parse_args(arguments) -def main(): +def main(): # pragma: no cover try: # TODO: Detect whether only legacy config is present. If so, inform the user about how to # upgrade, then exet. diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py index 8f645c0..77735bf 100644 --- a/borgmatic/commands/convert_config.py +++ b/borgmatic/commands/convert_config.py @@ -11,6 +11,7 @@ from borgmatic.config import convert, generate, legacy, validate DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config' +# TODO: Fold excludes into the YAML config file. DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' @@ -37,7 +38,7 @@ def parse_arguments(*arguments): return parser.parse_args(arguments) -def main(): +def main(): # pragma: no cover try: args = parse_arguments(*sys.argv[1:]) source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT) diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py index f326d6a..0f04740 100644 --- a/borgmatic/commands/generate_config.py +++ b/borgmatic/commands/generate_config.py @@ -29,7 +29,7 @@ def parse_arguments(*arguments): return parser.parse_args(arguments) -def main(): +def main(): # pragma: no cover try: args = parse_arguments(*sys.argv[1:]) diff --git a/borgmatic/tests/integration/commands/test_convert_config.py b/borgmatic/tests/integration/commands/test_convert_config.py new file mode 100644 index 0000000..b2bc32b --- /dev/null +++ b/borgmatic/tests/integration/commands/test_convert_config.py @@ -0,0 +1,14 @@ +from borgmatic.commands import convert_config as module + + +def test_parse_arguments_with_no_arguments_uses_defaults(): + parser = module.parse_arguments() + + assert parser.source_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME + assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME + +def test_parse_arguments_with_filename_arguments_overrides_defaults(): + parser = module.parse_arguments('--source', 'config', '--destination', 'config.yaml') + + assert parser.source_filename == 'config' + assert parser.destination_filename == 'config.yaml' diff --git a/borgmatic/tests/integration/commands/test_generate_config.py b/borgmatic/tests/integration/commands/test_generate_config.py new file mode 100644 index 0000000..b9b4db0 --- /dev/null +++ b/borgmatic/tests/integration/commands/test_generate_config.py @@ -0,0 +1,12 @@ +from borgmatic.commands import generate_config as module + + +def test_parse_arguments_with_no_arguments_uses_defaults(): + parser = module.parse_arguments() + + assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME + +def test_parse_arguments_with_filename_argument_overrides_defaults(): + parser = module.parse_arguments('--destination', 'config.yaml') + + assert parser.destination_filename == 'config.yaml' From fea97b514950fb96bf6dd716e7e0344f81bed041 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2017 09:43:25 -0700 Subject: [PATCH 132/189] Merge excludes into config file format. --- borgmatic/commands/convert_config.py | 33 ++++++++++---- borgmatic/config/convert.py | 19 +++++--- borgmatic/config/schema.yaml | 12 ++++++ .../integration/commands/test_borgmatic.py | 20 +++------ .../commands/test_convert_config.py | 43 ++++++++++++++++--- 5 files changed, 95 insertions(+), 32 deletions(-) diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py index 77735bf..2b99863 100644 --- a/borgmatic/commands/convert_config.py +++ b/borgmatic/commands/convert_config.py @@ -11,7 +11,6 @@ from borgmatic.config import convert, generate, legacy, validate DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config' -# TODO: Fold excludes into the YAML config file. DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' @@ -21,16 +20,27 @@ def parse_arguments(*arguments): Given command-line arguments with which this script was invoked, parse the arguments and return them as an ArgumentParser instance. ''' - parser = ArgumentParser(description='Convert a legacy INI-style borgmatic configuration file to YAML. Does not preserve comments.') + parser = ArgumentParser( + description=''' + Convert legacy INI-style borgmatic configuration and excludes files to a single YAML + configuration file. Note that this replaces any comments from the source files. + ''' + ) parser.add_argument( - '-s', '--source', - dest='source_filename', + '-s', '--source-config', + dest='source_config_filename', default=DEFAULT_SOURCE_CONFIG_FILENAME, help='Source INI-style configuration filename. Default: {}'.format(DEFAULT_SOURCE_CONFIG_FILENAME), ) parser.add_argument( - '-d', '--destination', - dest='destination_filename', + '-e', '--source-excludes', + dest='source_excludes_filename', + default=DEFAULT_SOURCE_EXCLUDES_FILENAME if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME) else None, + help='Excludes filename', + ) + parser.add_argument( + '-d', '--destination-config', + dest='destination_config_filename', default=DEFAULT_DESTINATION_CONFIG_FILENAME, help='Destination YAML configuration filename. Default: {}'.format(DEFAULT_DESTINATION_CONFIG_FILENAME), ) @@ -41,12 +51,17 @@ def parse_arguments(*arguments): def main(): # pragma: no cover try: args = parse_arguments(*sys.argv[1:]) - source_config = legacy.parse_configuration(args.source_filename, legacy.CONFIG_FORMAT) schema = yaml.round_trip_load(open(validate.schema_filename()).read()) + source_config = legacy.parse_configuration(args.source_config_filename, legacy.CONFIG_FORMAT) + source_excludes = ( + open(args.source_excludes_filename).read().splitlines() + if args.source_excludes_filename + else [] + ) - destination_config = convert.convert_legacy_parsed_config(source_config, schema) + destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema) - generate.write_configuration(args.destination_filename, destination_config) + generate.write_configuration(args.destination_config_filename, destination_config) # TODO: As a backstop, check that the written config can actually be read and parsed, and # that it matches the destination config data structure that was written. diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index 72b9ba3..11618e9 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -12,16 +12,15 @@ def _convert_section(source_section_config, section_schema): returned CommentedMap. ''' destination_section_config = yaml.comments.CommentedMap(source_section_config) - generate.add_comments_to_configuration(destination_section_config, section_schema, indent=generate.INDENT) return destination_section_config -def convert_legacy_parsed_config(source_config, schema): +def convert_legacy_parsed_config(source_config, source_excludes, schema): ''' - Given a legacy Parsed_config instance loaded from an INI-style config file, convert it to its - corresponding yaml.comments.CommentedMap representation in preparation for actual serialization - to YAML. + Given a legacy Parsed_config instance loaded from an INI-style config file and a list of exclude + patterns, convert them to a corresponding yaml.comments.CommentedMap representation in + preparation for serialization to a single YAML config file. Additionally, use the given schema as a source of helpful comments to include within the returned CommentedMap. @@ -31,11 +30,21 @@ def convert_legacy_parsed_config(source_config, schema): for section_name, section_config in source_config._asdict().items() ]) + # Split space-seperated values into actual lists, and merge in excludes. destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ') + destination_config['location']['exclude_patterns'] = source_excludes if source_config.consistency['checks']: destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') + # Add comments to each section, and then add comments to the fields in each section. generate.add_comments_to_configuration(destination_config, schema) + for section_name, section_config in destination_config.items(): + generate.add_comments_to_configuration( + section_config, + schema['map'][section_name], + indent=generate.INDENT, + ) + return destination_config diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 347dfee..97f13d9 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -30,6 +30,18 @@ map: type: scalar desc: Path to local or remote repository. example: user@backupserver:sourcehostname.borg + exclude_patterns: + seq: + - type: scalar + desc: | + Exclude patterns. Any paths matching these patterns are excluded from backups. + Globs are expanded. See + https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-patterns for + details. + example: + - '*.pyc' + - /home/*/.cache + - /etc/ssl storage: desc: | Repository storage options. See diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py index 69c2468..12afeed 100644 --- a/borgmatic/tests/integration/commands/test_borgmatic.py +++ b/borgmatic/tests/integration/commands/test_borgmatic.py @@ -1,5 +1,4 @@ import os -import sys from flexmock import flexmock import pytest @@ -14,7 +13,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME - assert parser.verbosity == None + assert parser.verbosity is None def test_parse_arguments_with_filename_arguments_overrides_defaults(): @@ -24,7 +23,7 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults(): assert parser.config_filename == 'myconfig' assert parser.excludes_filename == 'myexcludes' - assert parser.verbosity == None + assert parser.verbosity is None def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none(): @@ -33,8 +32,8 @@ def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_non parser = module.parse_arguments() assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME - assert parser.excludes_filename == None - assert parser.verbosity == None + assert parser.excludes_filename is None + assert parser.verbosity is None def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename(): @@ -44,7 +43,7 @@ def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename( assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME assert parser.excludes_filename == 'myexcludes' - assert parser.verbosity == None + assert parser.verbosity is None def test_parse_arguments_with_verbosity_flag_overrides_default(): @@ -59,11 +58,6 @@ def test_parse_arguments_with_verbosity_flag_overrides_default(): def test_parse_arguments_with_invalid_arguments_exits(): flexmock(os.path).should_receive('exists').and_return(True) - original_stderr = sys.stderr - sys.stderr = sys.stdout - try: - with pytest.raises(SystemExit): - module.parse_arguments('--posix-me-harder') - finally: - sys.stderr = original_stderr + with pytest.raises(SystemExit): + module.parse_arguments('--posix-me-harder') diff --git a/borgmatic/tests/integration/commands/test_convert_config.py b/borgmatic/tests/integration/commands/test_convert_config.py index b2bc32b..e126c12 100644 --- a/borgmatic/tests/integration/commands/test_convert_config.py +++ b/borgmatic/tests/integration/commands/test_convert_config.py @@ -1,14 +1,47 @@ +import os + +from flexmock import flexmock +import pytest + from borgmatic.commands import convert_config as module def test_parse_arguments_with_no_arguments_uses_defaults(): + flexmock(os.path).should_receive('exists').and_return(True) + parser = module.parse_arguments() - assert parser.source_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME - assert parser.destination_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME + assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME + assert parser.source_excludes_filename == module.DEFAULT_SOURCE_EXCLUDES_FILENAME + assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME + def test_parse_arguments_with_filename_arguments_overrides_defaults(): - parser = module.parse_arguments('--source', 'config', '--destination', 'config.yaml') + flexmock(os.path).should_receive('exists').and_return(True) - assert parser.source_filename == 'config' - assert parser.destination_filename == 'config.yaml' + parser = module.parse_arguments( + '--source-config', 'config', + '--source-excludes', 'excludes', + '--destination-config', 'config.yaml', + ) + + assert parser.source_config_filename == 'config' + assert parser.source_excludes_filename == 'excludes' + assert parser.destination_config_filename == 'config.yaml' + + +def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none(): + flexmock(os.path).should_receive('exists').and_return(False) + + parser = module.parse_arguments() + + assert parser.source_config_filename == module.DEFAULT_SOURCE_CONFIG_FILENAME + assert parser.source_excludes_filename is None + assert parser.destination_config_filename == module.DEFAULT_DESTINATION_CONFIG_FILENAME + + +def test_parse_arguments_with_invalid_arguments_exits(): + flexmock(os.path).should_receive('exists').and_return(True) + + with pytest.raises(SystemExit): + module.parse_arguments('--posix-me-harder') From 338b80903cd5f5a84913f48af8648d09e136a099 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2017 10:09:06 -0700 Subject: [PATCH 133/189] Fixing tests broken by excludes merging. --- borgmatic/tests/unit/config/test_convert.py | 27 ++++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py index 39fc385..a88ed2f 100644 --- a/borgmatic/tests/unit/config/test_convert.py +++ b/borgmatic/tests/unit/config/test_convert.py @@ -1,5 +1,7 @@ from collections import defaultdict, OrderedDict, namedtuple +from flexmock import flexmock + from borgmatic.config import convert as module @@ -7,18 +9,27 @@ Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): + flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) source_config = Parsed_config( location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]), storage=OrderedDict([('encryption_passphrase', 'supersecret')]), retention=OrderedDict([('keep_daily', 7)]), consistency=OrderedDict([('checks', 'repository')]), ) + source_excludes = ['/var'] schema = {'map': defaultdict(lambda: {'map': {}})} - destination_config = module.convert_legacy_parsed_config(source_config, schema) + destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema) assert destination_config == OrderedDict([ - ('location', OrderedDict([('source_directories', ['/home']), ('repository', 'hostname.borg')])), + ( + 'location', + OrderedDict([ + ('source_directories', ['/home']), + ('repository', 'hostname.borg'), + ('exclude_patterns', ['/var']), + ]), + ), ('storage', OrderedDict([('encryption_passphrase', 'supersecret')])), ('retention', OrderedDict([('keep_daily', 7)])), ('consistency', OrderedDict([('checks', ['repository'])])), @@ -26,18 +37,26 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): def test_convert_legacy_parsed_config_splits_space_separated_values(): + flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) source_config = Parsed_config( location=OrderedDict([('source_directories', '/home /etc')]), storage=OrderedDict(), retention=OrderedDict(), consistency=OrderedDict([('checks', 'repository archives')]), ) + source_excludes = ['/var'] schema = {'map': defaultdict(lambda: {'map': {}})} - destination_config = module.convert_legacy_parsed_config(source_config, schema) + destination_config = module.convert_legacy_parsed_config(source_config, source_excludes, schema) assert destination_config == OrderedDict([ - ('location', OrderedDict([('source_directories', ['/home', '/etc'])])), + ( + 'location', + OrderedDict([ + ('source_directories', ['/home', '/etc']), + ('exclude_patterns', ['/var']), + ]), + ), ('storage', OrderedDict()), ('retention', OrderedDict()), ('consistency', OrderedDict([('checks', ['repository', 'archives'])])), From 618e56b2a5f9b2cb381ea805e811cf1f8d76b1f8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2017 10:13:57 -0700 Subject: [PATCH 134/189] Display result of config upgrade. --- borgmatic/commands/convert_config.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py index 2b99863..eb97ec4 100644 --- a/borgmatic/commands/convert_config.py +++ b/borgmatic/commands/convert_config.py @@ -3,6 +3,7 @@ from argparse import ArgumentParser import os from subprocess import CalledProcessError import sys +import textwrap from ruamel import yaml @@ -48,6 +49,30 @@ def parse_arguments(*arguments): return parser.parse_args(arguments) +TEXT_WRAP_CHARACTERS = 80 + + +def display_result(args): # pragma: no cover + result_lines = textwrap.wrap( + 'Your borgmatic configuration has been upgraded. Please review the result in {}.'.format( + args.destination_config_filename + ), + TEXT_WRAP_CHARACTERS, + ) + + delete_lines = textwrap.wrap( + 'Once you are satisfied, you can safely delete {}{}.'.format( + args.source_config_filename, + ' and {}'.format(args.source_excludes_filename) if args.source_excludes_filename else '', + ), + TEXT_WRAP_CHARACTERS, + ) + + print('\n'.join(result_lines)) + print() + print('\n'.join(delete_lines)) + + def main(): # pragma: no cover try: args = parse_arguments(*sys.argv[1:]) @@ -65,6 +90,8 @@ def main(): # pragma: no cover # TODO: As a backstop, check that the written config can actually be read and parsed, and # that it matches the destination config data structure that was written. + + display_result(args) except (ValueError, OSError) as error: print(error, file=sys.stderr) sys.exit(1) From 5ff016238e0c15691221ccd5f330b09b0e5df8ee Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2017 10:37:11 -0700 Subject: [PATCH 135/189] Don't overwrite config files. And retain file permissions when upgrading config. --- borgmatic/commands/convert_config.py | 2 ++ borgmatic/config/generate.py | 4 ++++ borgmatic/tests/integration/config/test_generate.py | 12 ++++++++++++ 3 files changed, 18 insertions(+) diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py index eb97ec4..eb0bfa8 100644 --- a/borgmatic/commands/convert_config.py +++ b/borgmatic/commands/convert_config.py @@ -78,6 +78,7 @@ def main(): # pragma: no cover args = parse_arguments(*sys.argv[1:]) schema = yaml.round_trip_load(open(validate.schema_filename()).read()) source_config = legacy.parse_configuration(args.source_config_filename, legacy.CONFIG_FORMAT) + source_config_file_mode = os.stat(args.source_config_filename).st_mode source_excludes = ( open(args.source_excludes_filename).read().splitlines() if args.source_excludes_filename @@ -87,6 +88,7 @@ def main(): # pragma: no cover destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema) generate.write_configuration(args.destination_config_filename, destination_config) + os.chmod(args.destination_config_filename, source_config_file_mode) # TODO: As a backstop, check that the written config can actually be read and parsed, and # that it matches the destination config data structure that was written. diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 66d6c44..8b552d5 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -1,4 +1,5 @@ from collections import OrderedDict +import os from ruamel import yaml @@ -44,6 +45,9 @@ def write_configuration(config_filename, config): Given a target config filename and a config data structure of nested OrderedDicts, write out the config to file as YAML. ''' + if os.path.exists(config_filename): + raise FileExistsError('{} already exists. Aborting.'.format(config_filename)) + with open(config_filename, 'w') as config_file: config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)) diff --git a/borgmatic/tests/integration/config/test_generate.py b/borgmatic/tests/integration/config/test_generate.py index 89d8467..078367d 100644 --- a/borgmatic/tests/integration/config/test_generate.py +++ b/borgmatic/tests/integration/config/test_generate.py @@ -1,7 +1,9 @@ from io import StringIO +import os import sys from flexmock import flexmock +import pytest from borgmatic.config import generate as module @@ -15,12 +17,22 @@ def test_insert_newline_before_comment_does_not_raise(): def test_write_configuration_does_not_raise(): + flexmock(os.path).should_receive('exists').and_return(False) builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').and_return(StringIO()) module.write_configuration('config.yaml', {}) +def test_write_configuration_with_already_existing_file_raises(): + flexmock(os.path).should_receive('exists').and_return(True) + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').and_return(StringIO()) + + with pytest.raises(FileExistsError): + module.write_configuration('config.yaml', {}) + + def test_add_comments_to_configuration_does_not_raise(): # Ensure that it can deal with fields both in the schema and missing from the schema. config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)]) From ff28be77249bc28ba28006c8313fb470166a3d1b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2017 11:06:28 -0700 Subject: [PATCH 136/189] Documentation updates based on the new YAML configuration. --- README.md | 54 +++++++++++++++++++------- borgmatic/commands/generate_config.py | 5 +++ borgmatic/config/schema.yaml | 11 +++--- sample/config | 48 ----------------------- sample/config.yaml | 56 --------------------------- sample/excludes | 3 -- 6 files changed, 51 insertions(+), 126 deletions(-) delete mode 100644 sample/config delete mode 100644 sample/config.yaml delete mode 100644 sample/excludes diff --git a/README.md b/README.md index 5e69283..b1225c3 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,10 @@ location: # Path to local or remote repository. repository: user@backupserver:sourcehostname.borg + # Any paths matching these patterns are excluded from backups. + exclude_patterns: + - /home/*/.cache + retention: # Retention policy for how many backups to keep in each category. keep_daily: 7 @@ -37,16 +41,13 @@ consistency: - archives ``` -Additionally, exclude patterns can be specified in a separate excludes config -file, one pattern per line. - borgmatic is hosted at with [source code available](https://torsion.org/hg/borgmatic). It's also mirrored on [GitHub](https://github.com/witten/borgmatic) and [BitBucket](https://bitbucket.org/dhelfman/borgmatic) for convenience. -## Setup +## Installation To get up and running, follow the [Borg Quick Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create @@ -66,19 +67,45 @@ To install borgmatic, run the following command to download and install it: Make sure you're using Python 3, as borgmatic does not support Python 2. (You may have to use "pip3" instead of "pip".) -Then, download a [sample config -file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/config.yaml) and a -[sample excludes -file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/excludes). From the -directory where you downloaded them: +Then, generate a sample configuration file: - sudo mkdir /etc/borgmatic/ - sudo mv config.yaml excludes /etc/borgmatic/ + sudo generate-borgmatic-config -Lastly, modify the /etc files with your desired configuration. +This generates a sample configuration file at /etc/borgmatic/config.yaml (by +default). You should edit the file to suit your needs, as the values are just +representative. All fields are optional except where indicated, so feel free +to remove anything you don't need. -## Upgrading from atticmatic +## Upgrading + +In general, all you should need to do to upgrade borgmatic is run the +following: + + sudo pip install --upgrade borgmatic + +However, see below about special cases. + +### Upgrading from borgmatic 1.0.x + +borgmatic changed its configuration file format in version 1.1.0 from +INI-style to YAML. This better supports validation, and has a more natural way +to express lists of values. To upgrade your existing configuration, first +upgrade to the new version of borgmatic: + + sudo pip install --upgrade borgmatic + +Then, run: + + sudo upgrade-borgmatic-configuration + +That will generate a new YAML configuration file at /etc/borgmatic/config.yaml +(by default) using the values from both your existing configuration and +excludes files. The new version of borgmatic will consume the YAML +configuration file instead of the old one. + + +### Upgrading from atticmatic You can ignore this section if you're not an atticmatic user (the former name of borgmatic). @@ -98,6 +125,7 @@ from atticmatic to borgmatic. Simply run the following commands: That's it! borgmatic will continue using your /etc/borgmatic configuration files. + ## Usage You can run borgmatic and start a backup simply by invoking it without diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py index 0f04740..ba88ddf 100644 --- a/borgmatic/commands/generate_config.py +++ b/borgmatic/commands/generate_config.py @@ -34,6 +34,11 @@ def main(): # pragma: no cover args = parse_arguments(*sys.argv[1:]) generate.generate_sample_configuration(args.destination_filename, validate.schema_filename()) + + print('Generated a sample configuration file at {}.'.format(args.destination_filename)) + print() + print('Please edit the file to suit your needs. The values are just representative.') + print('All fields are optional except where indicated.') except (ValueError, OSError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 97f13d9..dfd0c55 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -12,7 +12,7 @@ map: required: True seq: - type: scalar - desc: List of source directories to backup. Globs are expanded. + desc: List of source directories to backup (required). Globs are expanded. example: - /home - /etc @@ -28,16 +28,15 @@ map: repository: required: True type: scalar - desc: Path to local or remote repository. + desc: Path to local or remote repository (required). example: user@backupserver:sourcehostname.borg exclude_patterns: seq: - type: scalar desc: | - Exclude patterns. Any paths matching these patterns are excluded from backups. - Globs are expanded. See - https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-patterns for - details. + Any paths matching these patterns are excluded from backups. Globs are expanded. + See https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-patterns + for details. example: - '*.pyc' - /home/*/.cache diff --git a/sample/config b/sample/config deleted file mode 100644 index e778aaa..0000000 --- a/sample/config +++ /dev/null @@ -1,48 +0,0 @@ -[location] -# Space-separated list of source directories to backup. -# Globs are expanded. -source_directories: /home /etc /var/log/syslog* - -# Stay in same file system (do not cross mount points). -#one_file_system: True - -# Alternate Borg remote executable (defaults to "borg"): -#remote_path: borg1 - -# Path to local or remote repository. -repository: user@backupserver:sourcehostname.borg - -[storage] -# Passphrase to unlock the encryption key with. Only use on repositories that -# were initialized with passphrase/repokey encryption. -#encryption_passphrase: foo - -# Type of compression to use when creating archives. See -# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create -# for details. Defaults to no compression. -#compression: lz4 - -# Umask to be used for borg create. -#umask: 0077 - -[retention] -# Retention policy for how many backups to keep in each category. See -# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details. -#keep_within: 3H -#keep_hourly: 24 -keep_daily: 7 -keep_weekly: 4 -keep_monthly: 6 -keep_yearly: 1 - -#prefix: sourcehostname - -[consistency] -# Space-separated list of consistency checks to run: "repository", "archives", -# or both. Defaults to both. Set to "disabled" to disable all consistency -# checks. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check -# for details. -checks: repository archives - -# Restrict the number of checked archives to the last n. -#check_last: 3 diff --git a/sample/config.yaml b/sample/config.yaml deleted file mode 100644 index daac230..0000000 --- a/sample/config.yaml +++ /dev/null @@ -1,56 +0,0 @@ -location: - # List of source directories to backup. Globs are expanded. - source_directories: - - /home - - /etc - - /var/log/syslog* - - # Stay in same file system (do not cross mount points). - #one_file_system: yes - - # Alternate Borg remote executable (defaults to "borg"): - #remote_path: borg1 - - # Path to local or remote repository. - repository: user@backupserver:sourcehostname.borg - -#storage: - # Passphrase to unlock the encryption key with. Only use on repositories - # that were initialized with passphrase/repokey encryption. Quote the value - # if it contains punctuation so it parses correctly. And backslash any - # quote or backslash literals as well. - #encryption_passphrase: "foo" - - # Type of compression to use when creating archives. See - # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create - # for details. Defaults to no compression. - #compression: lz4 - - # Umask to be used for borg create. - #umask: 0077 - -retention: - # Retention policy for how many backups to keep in each category. See - # https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for - # details. - #keep_within: 3H - #keep_hourly: 24 - keep_daily: 7 - keep_weekly: 4 - keep_monthly: 6 - keep_yearly: 1 - - # When pruning, only consider archive names starting with this prefix. - #prefix: sourcehostname - -consistency: - # List of consistency checks to run: "repository", "archives", or both. - # Defaults to both. Set to "disabled" to disable all consistency checks. - # See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check - # for details. - checks: - - repository - - archives - - # Restrict the number of checked archives to the last n. - #check_last: 3 diff --git a/sample/excludes b/sample/excludes deleted file mode 100644 index 7e81c88..0000000 --- a/sample/excludes +++ /dev/null @@ -1,3 +0,0 @@ -*.pyc -/home/*/.cache -/etc/ssl From b3d0fb0cee97ced1e2f3cbdd54917d06eddad317 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2017 15:20:50 -0700 Subject: [PATCH 137/189] When writing config, make containing directory if necessary. Also default to tighter permissions. --- borgmatic/commands/borgmatic.py | 2 +- borgmatic/commands/convert_config.py | 7 +++++-- borgmatic/config/generate.py | 11 +++++++++-- .../tests/integration/config/test_generate.py | 14 ++++++++++++-- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 1fcc582..a1d53e3 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -42,7 +42,7 @@ def parse_arguments(*arguments): def main(): # pragma: no cover try: # TODO: Detect whether only legacy config is present. If so, inform the user about how to - # upgrade, then exet. + # upgrade, then exit. args = parse_arguments(*sys.argv[1:]) config = parse_configuration(args.config_filename, schema_filename()) diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py index eb0bfa8..5e61a77 100644 --- a/borgmatic/commands/convert_config.py +++ b/borgmatic/commands/convert_config.py @@ -87,8 +87,11 @@ def main(): # pragma: no cover destination_config = convert.convert_legacy_parsed_config(source_config, source_excludes, schema) - generate.write_configuration(args.destination_config_filename, destination_config) - os.chmod(args.destination_config_filename, source_config_file_mode) + generate.write_configuration( + args.destination_config_filename, + destination_config, + mode=source_config_file_mode, + ) # TODO: As a backstop, check that the written config can actually be read and parsed, and # that it matches the destination config data structure that was written. diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 8b552d5..626747f 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -40,17 +40,24 @@ def _schema_to_sample_configuration(schema, level=0): return config -def write_configuration(config_filename, config): +def write_configuration(config_filename, config, mode=0o600): ''' Given a target config filename and a config data structure of nested OrderedDicts, write out the - config to file as YAML. + config to file as YAML. Create any containing directories as needed. ''' if os.path.exists(config_filename): raise FileExistsError('{} already exists. Aborting.'.format(config_filename)) + try: + os.makedirs(os.path.dirname(config_filename), mode=0o700) + except FileExistsError: + pass + with open(config_filename, 'w') as config_file: config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)) + os.chmod(config_filename, mode) + def add_comments_to_configuration(config, schema, indent=0): ''' diff --git a/borgmatic/tests/integration/config/test_generate.py b/borgmatic/tests/integration/config/test_generate.py index 078367d..048315a 100644 --- a/borgmatic/tests/integration/config/test_generate.py +++ b/borgmatic/tests/integration/config/test_generate.py @@ -18,21 +18,31 @@ def test_insert_newline_before_comment_does_not_raise(): def test_write_configuration_does_not_raise(): flexmock(os.path).should_receive('exists').and_return(False) + flexmock(os).should_receive('makedirs') builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').and_return(StringIO()) + flexmock(os).should_receive('chmod') module.write_configuration('config.yaml', {}) def test_write_configuration_with_already_existing_file_raises(): flexmock(os.path).should_receive('exists').and_return(True) - builtins = flexmock(sys.modules['builtins']) - builtins.should_receive('open').and_return(StringIO()) with pytest.raises(FileExistsError): module.write_configuration('config.yaml', {}) +def test_write_configuration_with_already_existing_directory_does_not_raise(): + flexmock(os.path).should_receive('exists').and_return(False) + flexmock(os).should_receive('makedirs').and_raise(FileExistsError) + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').and_return(StringIO()) + flexmock(os).should_receive('chmod') + + module.write_configuration('config.yaml', {}) + + def test_add_comments_to_configuration_does_not_raise(): # Ensure that it can deal with fields both in the schema and missing from the schema. config = module.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)]) From 8bf07e476620f47597b5ff6962b9387237e85f82 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2017 16:06:02 -0700 Subject: [PATCH 138/189] Provide helpful message when borgmatic is run with only legacy config present. --- README.md | 2 +- borgmatic/commands/borgmatic.py | 9 +++--- borgmatic/config/convert.py | 29 +++++++++++++++++++ borgmatic/tests/unit/config/test_convert.py | 31 +++++++++++++++++++++ 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b1225c3..6c25de2 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ upgrade to the new version of borgmatic: Then, run: - sudo upgrade-borgmatic-configuration + sudo upgrade-borgmatic-config That will generate a new YAML configuration file at /etc/borgmatic/config.yaml (by default) using the values from both your existing configuration and diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index a1d53e3..0299da6 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -5,9 +5,10 @@ from subprocess import CalledProcessError import sys from borgmatic import borg -from borgmatic.config.validate import parse_configuration, schema_filename +from borgmatic.config import convert, validate +LEGACY_CONFIG_FILENAME = '/etc/borgmatic/config' DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' @@ -41,11 +42,9 @@ def parse_arguments(*arguments): def main(): # pragma: no cover try: - # TODO: Detect whether only legacy config is present. If so, inform the user about how to - # upgrade, then exit. - args = parse_arguments(*sys.argv[1:]) - config = parse_configuration(args.config_filename, schema_filename()) + convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename) + config = validate.parse_configuration(args.config_filename, validate.schema_filename()) repository = config.location['repository'] remote_path = config.location.get('remote_path') diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index 11618e9..bdf9eb2 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -1,3 +1,5 @@ +import os + from ruamel import yaml from borgmatic.config import generate @@ -48,3 +50,30 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema): ) return destination_config + + +class LegacyConfigurationNotUpgraded(FileNotFoundError): + def __init__(self): + super(LegacyConfigurationNotUpgraded, self).__init__( + '''borgmatic changed its configuration file format in version 1.1.0 from INI-style +to YAML. This better supports validation, and has a more natural way to express +lists of values. To upgrade your existing configuration, run: + + sudo upgrade-borgmatic-config + +That will generate a new YAML configuration file at /etc/borgmatic/config.yaml +(by default) using the values from both your existing configuration and excludes +files. The new version of borgmatic will consume the YAML configuration file +instead of the old one.''' + ) + + +def guard_configuration_upgraded(source_config_filename, destination_config_filename): + ''' + If legacy souce configuration exists but destination upgraded config doesn't, raise + LegacyConfigurationNotUpgraded. + + The idea is that we want to alert the user about upgrading their config if they haven't already. + ''' + if os.path.exists(source_config_filename) and not os.path.exists(destination_config_filename): + raise LegacyConfigurationNotUpgraded() diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py index a88ed2f..2e5849b 100644 --- a/borgmatic/tests/unit/config/test_convert.py +++ b/borgmatic/tests/unit/config/test_convert.py @@ -1,6 +1,8 @@ from collections import defaultdict, OrderedDict, namedtuple +import os from flexmock import flexmock +import pytest from borgmatic.config import convert as module @@ -61,3 +63,32 @@ def test_convert_legacy_parsed_config_splits_space_separated_values(): ('retention', OrderedDict()), ('consistency', OrderedDict([('checks', ['repository', 'archives'])])), ]) + + +def test_guard_configuration_upgraded_raises_when_only_source_config_present(): + flexmock(os.path).should_receive('exists').with_args('config').and_return(True) + flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) + + with pytest.raises(module.LegacyConfigurationNotUpgraded): + module.guard_configuration_upgraded('config', 'config.yaml') + + +def test_guard_configuration_upgraded_does_not_raise_when_only_destination_config_present(): + flexmock(os.path).should_receive('exists').with_args('config').and_return(False) + flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True) + + module.guard_configuration_upgraded('config', 'config.yaml') + + +def test_guard_configuration_upgraded_does_not_raise_when_both_configs_present(): + flexmock(os.path).should_receive('exists').with_args('config').and_return(True) + flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True) + + module.guard_configuration_upgraded('config', 'config.yaml') + + +def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present(): + flexmock(os.path).should_receive('exists').with_args('config').and_return(False) + flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) + + module.guard_configuration_upgraded('config', 'config.yaml') From 0691cda46fe74705e8e3855fd4d926fb74e583ea Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2017 16:07:07 -0700 Subject: [PATCH 139/189] Mention generate-borgmatic-config in changelog. --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 3209b41..e3394d4 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 1.1.0.dev0 * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade. + * Added generate-borgmatic-config command for initial config creation. * Dropped Python 2 support. Now Python 3 only. * #18: Fix for README mention of sample files not included in package. * #22: Sample files for triggering borgmatic from a systemd timer. From 8ef6c6fcbe222471eb8efed653b296fab367ce57 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2017 16:25:13 -0700 Subject: [PATCH 140/189] Bail if "--excludes" argument is provided, as it's now deprecated in favor of configuration file. --- borgmatic/commands/borgmatic.py | 5 ++- borgmatic/config/convert.py | 18 +++++++++++ .../integration/commands/test_borgmatic.py | 32 ++----------------- borgmatic/tests/unit/config/test_convert.py | 9 ++++++ 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 0299da6..33b156a 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -28,8 +28,7 @@ def parse_arguments(*arguments): parser.add_argument( '--excludes', dest='excludes_filename', - default=DEFAULT_EXCLUDES_FILENAME if os.path.exists(DEFAULT_EXCLUDES_FILENAME) else None, - help='Excludes filename', + help='Excludes filename, deprecated in favor of excludes_patterns within configuration', ) parser.add_argument( '-v', '--verbosity', @@ -46,7 +45,7 @@ def main(): # pragma: no cover convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename) config = validate.parse_configuration(args.config_filename, validate.schema_filename()) repository = config.location['repository'] - remote_path = config.location.get('remote_path') + remote_path = config.location['remote_path'] borg.initialize(config.storage) borg.create_archive( diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index bdf9eb2..1aaf7aa 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -77,3 +77,21 @@ def guard_configuration_upgraded(source_config_filename, destination_config_file ''' if os.path.exists(source_config_filename) and not os.path.exists(destination_config_filename): raise LegacyConfigurationNotUpgraded() + + +class LegacyExcludesFilenamePresent(FileNotFoundError): + def __init__(self): + super(LegacyExcludesFilenamePresent, self).__init__( + '''borgmatic changed its configuration file format in version 1.1.0 from INI-style +to YAML. This better supports validation, and has a more natural way to express +lists of values. The new configuration file incorporates excludes, so you no +longer need to provide an excludes filename on the command-line with an +"--excludes" argument. + +Please remove the "--excludes" argument and run borgmatic again.''' + ) + + +def guard_excludes_filename_omitted(excludes_filename): + if excludes_filename != None: + raise LegacyExcludesFilenamePresent() diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py index 12afeed..f1cc43a 100644 --- a/borgmatic/tests/integration/commands/test_borgmatic.py +++ b/borgmatic/tests/integration/commands/test_borgmatic.py @@ -7,18 +7,14 @@ from borgmatic.commands import borgmatic as module def test_parse_arguments_with_no_arguments_uses_defaults(): - flexmock(os.path).should_receive('exists').and_return(True) - parser = module.parse_arguments() assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME - assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME + assert parser.excludes_filename == None assert parser.verbosity is None def test_parse_arguments_with_filename_arguments_overrides_defaults(): - flexmock(os.path).should_receive('exists').and_return(True) - parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes') assert parser.config_filename == 'myconfig' @@ -26,38 +22,14 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults(): assert parser.verbosity is None -def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none(): - flexmock(os.path).should_receive('exists').and_return(False) - - parser = module.parse_arguments() - - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME - assert parser.excludes_filename is None - assert parser.verbosity is None - - -def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename(): - flexmock(os.path).should_receive('exists').and_return(False) - - parser = module.parse_arguments('--excludes', 'myexcludes') - - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME - assert parser.excludes_filename == 'myexcludes' - assert parser.verbosity is None - - def test_parse_arguments_with_verbosity_flag_overrides_default(): - flexmock(os.path).should_receive('exists').and_return(True) - parser = module.parse_arguments('--verbosity', '1') assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME - assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME + assert parser.excludes_filename == None assert parser.verbosity == 1 def test_parse_arguments_with_invalid_arguments_exits(): - flexmock(os.path).should_receive('exists').and_return(True) - with pytest.raises(SystemExit): module.parse_arguments('--posix-me-harder') diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py index 2e5849b..39f0cee 100644 --- a/borgmatic/tests/unit/config/test_convert.py +++ b/borgmatic/tests/unit/config/test_convert.py @@ -92,3 +92,12 @@ def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) module.guard_configuration_upgraded('config', 'config.yaml') + + +def test_guard_excludes_filename_omitted_raises_when_filename_provided(): + with pytest.raises(module.LegacyExcludesFilenamePresent): + module.guard_excludes_filename_omitted(excludes_filename='/etc/borgmatic/excludes') + + +def test_guard_excludes_filename_omitted_does_not_raise_when_filename_not_provided(): + module.guard_excludes_filename_omitted(excludes_filename=None) From 41d202c2e7e4b4c6c9c987f81f30bdacd0c48b13 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Jul 2017 16:26:32 -0700 Subject: [PATCH 141/189] TODO about using the new exclude_patterns. --- borgmatic/commands/borgmatic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 33b156a..c2083d9 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -28,7 +28,7 @@ def parse_arguments(*arguments): parser.add_argument( '--excludes', dest='excludes_filename', - help='Excludes filename, deprecated in favor of excludes_patterns within configuration', + help='Excludes filename, deprecated in favor of exclude_patterns within configuration', ) parser.add_argument( '-v', '--verbosity', @@ -48,6 +48,7 @@ def main(): # pragma: no cover remote_path = config.location['remote_path'] borg.initialize(config.storage) + # TODO: Use the new exclude_patterns. borg.create_archive( args.excludes_filename, args.verbosity, config.storage, **config.location ) From edb54b300bac377fda0e8a48747edea06cfd46b5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 20:11:49 -0700 Subject: [PATCH 142/189] Fixing up borg module to deal with new parsed config file structures. --- borgmatic/borg.py | 59 +++++++++++------ borgmatic/commands/borgmatic.py | 5 +- borgmatic/tests/unit/test_borg.py | 103 ++++++++++++++++-------------- 3 files changed, 95 insertions(+), 72 deletions(-) diff --git a/borgmatic/borg.py b/borgmatic/borg.py index f4c5ba9..22d3032 100644 --- a/borgmatic/borg.py +++ b/borgmatic/borg.py @@ -1,10 +1,11 @@ from datetime import datetime +import glob +import itertools import os -import re import platform +import re import subprocess -from glob import glob -from itertools import chain +import tempfile from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS @@ -22,18 +23,38 @@ def initialize(storage_config, command=COMMAND): os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase +def _write_exclude_file(exclude_patterns=None): + ''' + Given a sequence of exclude patterns, write them to a named temporary file and return it. Return + None if no patterns are provided. + ''' + if not exclude_patterns: + return None + + exclude_file = tempfile.NamedTemporaryFile('w') + exclude_file.write('\n'.join(exclude_patterns)) + exclude_file.flush() + + return exclude_file + + def create_archive( - excludes_filename, verbosity, storage_config, source_directories, repository, command=COMMAND, - one_file_system=None, remote_path=None, + verbosity, storage_config, source_directories, repository, exclude_patterns=None, + command=COMMAND, one_file_system=None, remote_path=None, ): ''' - Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated - list of source directories, a local or remote repository path, and a command to run, create an - attic archive. + Given a vebosity flag, a storage config dict, a list of source directories, a local or remote + repository path, a list of exclude patterns, and a command to run, create an attic archive. ''' - sources = re.split('\s+', source_directories) - sources = tuple(chain.from_iterable(glob(x) or [x] for x in sources)) - exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else () + sources = tuple( + itertools.chain.from_iterable( + glob.glob(directory) or [directory] + for directory in source_directories + ) + ) + + exclude_file = _write_exclude_file(exclude_patterns) + exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else () compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () umask = storage_config.get('umask', None) @@ -109,12 +130,11 @@ DEFAULT_CHECKS = ('repository', 'archives') def _parse_checks(consistency_config): ''' - Given a consistency config with a space-separated "checks" option, transform it to a tuple of - named checks to run. + Given a consistency config with a "checks" list, transform it to a tuple of named checks to run. For example, given a retention config of: - {'checks': 'repository archives'} + {'checks': ['repository', 'archives']} This will be returned as: @@ -123,14 +143,11 @@ def _parse_checks(consistency_config): If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string "disabled", return an empty tuple, meaning that no checks should be run. ''' - checks = consistency_config.get('checks', '').strip() - if not checks: - return DEFAULT_CHECKS + checks = consistency_config.get('checks', []) + if checks == ['disabled']: + return () - return tuple( - check for check in consistency_config['checks'].split(' ') - if check.lower() not in ('disabled', '') - ) + return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS def _make_check_flags(checks, check_last=None): diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index c2083d9..9bde8bc 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -48,10 +48,7 @@ def main(): # pragma: no cover remote_path = config.location['remote_path'] borg.initialize(config.storage) - # TODO: Use the new exclude_patterns. - borg.create_archive( - args.excludes_filename, args.verbosity, config.storage, **config.location - ) + borg.create_archive(args.verbosity, config.storage, **config.location) borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path) borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path) except (ValueError, OSError, CalledProcessError) as error: diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 3c2aa1c..57e75a3 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -30,6 +30,20 @@ def test_initialize_without_passphrase_should_not_set_environment(): finally: os.environ = orig_environ +def test_write_exclude_file_does_not_raise(): + temporary_file = flexmock( + name='filename', + write=lambda mode: None, + flush=lambda: None, + ) + flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file) + + module._write_exclude_file(['exclude']) + + +def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise(): + module._write_exclude_file([]) + def insert_subprocess_mock(check_call_command, **kwargs): subprocess = flexmock(STDOUT=STDOUT) @@ -53,110 +67,100 @@ def insert_datetime_mock(): ).mock -CREATE_COMMAND_WITHOUT_EXCLUDES = ('borg', 'create', 'repo::host-now', 'foo', 'bar') -CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes') +CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar') def test_create_archive_should_call_borg_with_parameters(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) -def test_create_archive_with_two_spaces_in_source_directories(): - insert_subprocess_mock(CREATE_COMMAND) +def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): + flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes')) + insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes')) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=['exclude'], verbosity=None, storage_config={}, - source_directories='foo bar', - repository='repo', - command='borg', - ) - - -def test_create_archive_with_none_excludes_filename_should_call_borg_without_excludes(): - insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES) - insert_platform_mock() - insert_datetime_mock() - - module.create_archive( - excludes_filename=None, - verbosity=None, - storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',)) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=VERBOSITY_SOME, storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats')) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=VERBOSITY_LOTS, storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) def test_create_archive_with_compression_should_call_borg_with_compression_parameters(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=None, storage_config={'compression': 'rle'}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', one_file_system=True, @@ -164,15 +168,16 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1')) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', remote_path='borg1', @@ -180,63 +185,67 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) insert_platform_mock() insert_datetime_mock() module.create_archive( - excludes_filename='excludes', + exclude_patterns=None, verbosity=None, storage_config={'umask': 740}, - source_directories='foo bar', + source_directories=['foo', 'bar'], repository='repo', command='borg', ) def test_create_archive_with_source_directories_glob_expands(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() - flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) + flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( - excludes_filename=None, + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo*', + source_directories=['foo*'], repository='repo', command='borg', ) def test_create_archive_with_non_matching_source_directories_glob_passes_through(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*')) insert_platform_mock() insert_datetime_mock() - flexmock(module).should_receive('glob').with_args('foo*').and_return([]) + flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([]) module.create_archive( - excludes_filename=None, + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo*', + source_directories=['foo*'], repository='repo', command='borg', ) def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): + flexmock(module).should_receive('_write_exclude_file') insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() - flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) + flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( - excludes_filename=None, + exclude_patterns=None, verbosity=None, storage_config={}, - source_directories='foo*', + source_directories=['foo*'], repository='repo', command='borg', ) @@ -329,7 +338,7 @@ def test_prune_archive_with_remote_path_should_call_borg_with_remote_path_parame def test_parse_checks_returns_them_as_tuple(): - checks = module._parse_checks({'checks': 'foo disabled bar'}) + checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']}) assert checks == ('foo', 'bar') @@ -341,13 +350,13 @@ def test_parse_checks_with_missing_value_returns_defaults(): def test_parse_checks_with_blank_value_returns_defaults(): - checks = module._parse_checks({'checks': ''}) + checks = module._parse_checks({'checks': []}) assert checks == module.DEFAULT_CHECKS def test_parse_checks_with_disabled_returns_no_checks(): - checks = module._parse_checks({'checks': 'disabled'}) + checks = module._parse_checks({'checks': ['disabled']}) assert checks == () From 8bfffd8cf7c012a4e7ae899c9a8b79e2cba314f2 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 20:31:26 -0700 Subject: [PATCH 143/189] Removing TODO that basically entails testing ruamel.yaml round-tripping, which in theory already has its own tests. --- borgmatic/commands/convert_config.py | 3 --- borgmatic/config/generate.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py index 5e61a77..ca001e0 100644 --- a/borgmatic/commands/convert_config.py +++ b/borgmatic/commands/convert_config.py @@ -93,9 +93,6 @@ def main(): # pragma: no cover mode=source_config_file_mode, ) - # TODO: As a backstop, check that the written config can actually be read and parsed, and - # that it matches the destination config data structure that was written. - display_result(args) except (ValueError, OSError) as error: print(error, file=sys.stderr) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 626747f..1a6d1a8 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -50,7 +50,7 @@ def write_configuration(config_filename, config, mode=0o600): try: os.makedirs(os.path.dirname(config_filename), mode=0o700) - except FileExistsError: + except (FileExistsError, FileNotFoundError): pass with open(config_filename, 'w') as config_file: From 919d7573c3059e1b519fbd4b1d27f497ed414167 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 20:52:29 -0700 Subject: [PATCH 144/189] Upgrading instructions to super clarify Python 3 upgrade. --- README.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6c25de2..c87c2f7 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ To install borgmatic, run the following command to download and install it: sudo pip install --upgrade borgmatic Make sure you're using Python 3, as borgmatic does not support Python 2. (You -may have to use "pip3" instead of "pip".) +may have to use "pip3" or similar instead of "pip".) Then, generate a sample configuration file: @@ -84,8 +84,12 @@ following: sudo pip install --upgrade borgmatic +(You may have to use "pip3" or similar instead of "pip", so Python 3 gets +used.) + However, see below about special cases. + ### Upgrading from borgmatic 1.0.x borgmatic changed its configuration file format in version 1.1.0 from @@ -93,9 +97,22 @@ INI-style to YAML. This better supports validation, and has a more natural way to express lists of values. To upgrade your existing configuration, first upgrade to the new version of borgmatic: +As of version 1.1.0, borgmatic no longer supports Python 2. If you were +already running borgmatic with Python 3, then you can simply upgrade borgmatic +in-place: + sudo pip install --upgrade borgmatic -Then, run: +But if you were running borgmatic with Python 2, uninstall and reinstall instead: + + sudo pip uninstall borgmatic + sudo pip3 install borgmatic + +The pip binary names for different versions of Python can differ, so the above +commands may need some tweaking to work on your machine. + + +Once borgmatic is upgraded, run: sudo upgrade-borgmatic-config From 3cccac8cb17a908bf137d3db7e67e70f2be57759 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 21:07:09 -0700 Subject: [PATCH 145/189] Mentioning libyaml compile errors in troubleshooting. --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index c87c2f7..feebbdc 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,17 @@ This should make the client keep the connection alive while validating backups. +### libyaml compilation errors + +borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally +use a C YAML library (libyaml) if present. But if it's not installed, then +when installing or upgrading borgmatic, you may see errors about compiling the +YAML library. If so, not to worry. borgmatic should install and function +correctly even without the C YAML library. And borgmatic won't be any faster +with the C library present, so you don't need to go out of your way to install +it. + + ## Issues and feedback Got an issue or an idea for a feature enhancement? Check out the [borgmatic From 6af53d11636826abcf105188d2d8b08ae49e8100 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 21:19:26 -0700 Subject: [PATCH 146/189] Fixing gets on config group names. --- borgmatic/commands/borgmatic.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 9bde8bc..c37b891 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -44,13 +44,17 @@ def main(): # pragma: no cover args = parse_arguments(*sys.argv[1:]) convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename) config = validate.parse_configuration(args.config_filename, validate.schema_filename()) - repository = config.location['repository'] - remote_path = config.location['remote_path'] + repository = config['location']['repository'] + remote_path = config['location']['remote_path'] + (storage, retention, consistency) = ( + config.get(group_name, {}) + for group_name in ('storage', 'retention', 'consistency') + ) - borg.initialize(config.storage) - borg.create_archive(args.verbosity, config.storage, **config.location) - borg.prune_archives(args.verbosity, repository, config.retention, remote_path=remote_path) - borg.check_archives(args.verbosity, repository, config.consistency, remote_path=remote_path) + borg.initialize(storage) + borg.create_archive(args.verbosity, storage, **config['location']) + borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) + borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) From f5abe05ce96c1645bb9f0a68dbfbe3da723bb4c0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 21:20:48 -0700 Subject: [PATCH 147/189] Instructions to make cron file executable. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index feebbdc..0d25936 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,7 @@ file](https://torsion.org/hg/borgmatic/raw-file/tip/sample/cron/borgmatic). Then, from the directory where you downloaded it: sudo mv borgmatic /etc/cron.d/borgmatic + sudo chmod +x /etc/cron.d/borgmatic You can modify the cron file if you'd like to run borgmatic more or less frequently. From e5c12fc81c982d6e77ba7c0858284326b07ca79e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 21:23:01 -0700 Subject: [PATCH 148/189] Mentioning test coverage addition in NEWS. --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index e3394d4..0941a1c 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,7 @@ * Dropped Python 2 support. Now Python 3 only. * #18: Fix for README mention of sample files not included in package. * #22: Sample files for triggering borgmatic from a systemd timer. + * Enabled test coverage output during tox runs. * Added logo. 1.0.3 From cd8ceccfafbdba6536cefce660115954ee122dd3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 21:50:29 -0700 Subject: [PATCH 149/189] To free up space, now pruning backups prior to creating a new backup. --- NEWS | 1 + borgmatic/commands/borgmatic.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 0941a1c..47e5361 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,7 @@ * Dropped Python 2 support. Now Python 3 only. * #18: Fix for README mention of sample files not included in package. * #22: Sample files for triggering borgmatic from a systemd timer. + * To free up space, now pruning backups prior to creating a new backup. * Enabled test coverage output during tox runs. * Added logo. diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index c37b891..90fccca 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -52,8 +52,8 @@ def main(): # pragma: no cover ) borg.initialize(storage) - borg.create_archive(args.verbosity, storage, **config['location']) borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) + borg.create_archive(args.verbosity, storage, **config['location']) borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) From 90a0d3b1e02d6d7dc7d33065a69360215817d769 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 22:17:37 -0700 Subject: [PATCH 150/189] Renaming group to section for consistency. --- borgmatic/commands/borgmatic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 90fccca..f1f9c31 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -47,8 +47,8 @@ def main(): # pragma: no cover repository = config['location']['repository'] remote_path = config['location']['remote_path'] (storage, retention, consistency) = ( - config.get(group_name, {}) - for group_name in ('storage', 'retention', 'consistency') + config.get(section_name, {}) + for section_name in ('storage', 'retention', 'consistency') ) borg.initialize(storage) From ee3edeaac2239b9cefb67ced6ea3f6a6f0c7c27e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 22:56:46 -0700 Subject: [PATCH 151/189] Support for backing up to multiple repositories. --- NEWS | 1 + README.md | 5 +- borgmatic/borg.py | 10 +- borgmatic/commands/borgmatic.py | 20 ++-- borgmatic/config/convert.py | 9 +- borgmatic/config/schema.yaml | 12 +- .../tests/integration/config/test_validate.py | 13 ++- borgmatic/tests/unit/config/test_convert.py | 5 +- borgmatic/tests/unit/test_borg.py | 103 ++++++++++++------ 9 files changed, 115 insertions(+), 63 deletions(-) diff --git a/NEWS b/NEWS index 47e5361..4473170 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,7 @@ * Dropped Python 2 support. Now Python 3 only. * #18: Fix for README mention of sample files not included in package. * #22: Sample files for triggering borgmatic from a systemd timer. + * Support for backing up to multiple repositories. * To free up space, now pruning backups prior to creating a new backup. * Enabled test coverage output during tox runs. * Added logo. diff --git a/README.md b/README.md index 0d25936..0bd5cda 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,9 @@ location: - /etc - /var/log/syslog* - # Path to local or remote repository. - repository: user@backupserver:sourcehostname.borg + # Paths to local or remote repositories. + repositories: + - user@backupserver:sourcehostname.borg # Any paths matching these patterns are excluded from backups. exclude_patterns: diff --git a/borgmatic/borg.py b/borgmatic/borg.py index 22d3032..f42353d 100644 --- a/borgmatic/borg.py +++ b/borgmatic/borg.py @@ -39,8 +39,7 @@ def _write_exclude_file(exclude_patterns=None): def create_archive( - verbosity, storage_config, source_directories, repository, exclude_patterns=None, - command=COMMAND, one_file_system=None, remote_path=None, + verbosity, repository, location_config, storage_config, command=COMMAND, ): ''' Given a vebosity flag, a storage config dict, a list of source directories, a local or remote @@ -49,17 +48,18 @@ def create_archive( sources = tuple( itertools.chain.from_iterable( glob.glob(directory) or [directory] - for directory in source_directories + for directory in location_config['source_directories'] ) ) - exclude_file = _write_exclude_file(exclude_patterns) + exclude_file = _write_exclude_file(location_config.get('exclude_patterns')) exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else () compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () umask = storage_config.get('umask', None) umask_flags = ('--umask', str(umask)) if umask else () - one_file_system_flags = ('--one-file-system',) if one_file_system else () + one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else () + remote_path = location_config.get('remote_path') remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { VERBOSITY_SOME: ('--info', '--stats',), diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index f1f9c31..a5f39a0 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -44,17 +44,23 @@ def main(): # pragma: no cover args = parse_arguments(*sys.argv[1:]) convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename) config = validate.parse_configuration(args.config_filename, validate.schema_filename()) - repository = config['location']['repository'] - remote_path = config['location']['remote_path'] - (storage, retention, consistency) = ( + (location, storage, retention, consistency) = ( config.get(section_name, {}) - for section_name in ('storage', 'retention', 'consistency') + for section_name in ('location', 'storage', 'retention', 'consistency') ) + remote_path = location.get('remote_path') borg.initialize(storage) - borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) - borg.create_archive(args.verbosity, storage, **config['location']) - borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) + + for repository in location['repositories']: + borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) + borg.create_archive( + args.verbosity, + repository, + location, + storage, + ) + borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index 1aaf7aa..832d7d3 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -32,9 +32,12 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema): for section_name, section_config in source_config._asdict().items() ]) - # Split space-seperated values into actual lists, and merge in excludes. - destination_config['location']['source_directories'] = source_config.location['source_directories'].split(' ') - destination_config['location']['exclude_patterns'] = source_excludes + # Split space-seperated values into actual lists, make "repository" into a list, and merge in + # excludes. + location = destination_config['location'] + location['source_directories'] = source_config.location['source_directories'].split(' ') + location['repositories'] = [location.pop('repository')] + location['exclude_patterns'] = source_excludes if source_config.consistency['checks']: destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index dfd0c55..64fed07 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -25,11 +25,15 @@ map: type: scalar desc: Alternate Borg remote executable. Defaults to "borg". example: borg1 - repository: + repositories: required: True - type: scalar - desc: Path to local or remote repository (required). - example: user@backupserver:sourcehostname.borg + seq: + - type: scalar + desc: | + Paths to local or remote repositories (required). Multiple repositories are + backed up to in sequence. + example: + - user@backupserver:sourcehostname.borg exclude_patterns: seq: - type: scalar diff --git a/borgmatic/tests/integration/config/test_validate.py b/borgmatic/tests/integration/config/test_validate.py index 90e223d..9b63ccc 100644 --- a/borgmatic/tests/integration/config/test_validate.py +++ b/borgmatic/tests/integration/config/test_validate.py @@ -35,7 +35,8 @@ def test_parse_configuration_transforms_file_into_mapping(): - /home - /etc - repository: hostname.borg + repositories: + - hostname.borg retention: keep_daily: 7 @@ -50,7 +51,7 @@ def test_parse_configuration_transforms_file_into_mapping(): result = module.parse_configuration('config.yaml', 'schema.yaml') assert result == { - 'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'}, + 'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']}, 'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}, } @@ -65,7 +66,8 @@ def test_parse_configuration_passes_through_quoted_punctuation(): source_directories: - /home - repository: "{}.borg" + repositories: + - "{}.borg" '''.format(escaped_punctuation) ) @@ -74,7 +76,7 @@ def test_parse_configuration_passes_through_quoted_punctuation(): assert result == { 'location': { 'source_directories': ['/home'], - 'repository': '{}.borg'.format(string.punctuation), + 'repositories': ['{}.borg'.format(string.punctuation)], }, } @@ -105,7 +107,8 @@ def test_parse_configuration_raises_for_validation_error(): ''' location: source_directories: yes - repository: hostname.borg + repositories: + - hostname.borg ''' ) diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py index 39f0cee..827ecdb 100644 --- a/borgmatic/tests/unit/config/test_convert.py +++ b/borgmatic/tests/unit/config/test_convert.py @@ -28,7 +28,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): 'location', OrderedDict([ ('source_directories', ['/home']), - ('repository', 'hostname.borg'), + ('repositories', ['hostname.borg']), ('exclude_patterns', ['/var']), ]), ), @@ -41,7 +41,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): def test_convert_legacy_parsed_config_splits_space_separated_values(): flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) source_config = Parsed_config( - location=OrderedDict([('source_directories', '/home /etc')]), + location=OrderedDict([('source_directories', '/home /etc'), ('repository', 'hostname.borg')]), storage=OrderedDict(), retention=OrderedDict(), consistency=OrderedDict([('checks', 'repository archives')]), @@ -56,6 +56,7 @@ def test_convert_legacy_parsed_config_splits_space_separated_values(): 'location', OrderedDict([ ('source_directories', ['/home', '/etc']), + ('repositories', ['hostname.borg']), ('exclude_patterns', ['/var']), ]), ), diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 57e75a3..36cc312 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -77,11 +77,14 @@ def test_create_archive_should_call_borg_with_parameters(): insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', ) @@ -93,11 +96,14 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): insert_datetime_mock() module.create_archive( - exclude_patterns=['exclude'], verbosity=None, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': ['exclude'], + }, + storage_config={}, command='borg', ) @@ -109,11 +115,14 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=VERBOSITY_SOME, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', ) @@ -125,11 +134,14 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=VERBOSITY_LOTS, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', ) @@ -141,11 +153,14 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={'compression': 'rle'}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'compression': 'rle'}, command='borg', ) @@ -157,13 +172,16 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'one_file_system': True, + 'exclude_patterns': None, + }, + storage_config={}, command='borg', - one_file_system=True, ) @@ -174,13 +192,16 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'remote_path': 'borg1', + 'exclude_patterns': None, + }, + storage_config={}, command='borg', - remote_path='borg1', ) @@ -191,11 +212,14 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): insert_datetime_mock() module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={'umask': 740}, - source_directories=['foo', 'bar'], repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'umask': 740}, command='borg', ) @@ -208,11 +232,14 @@ def test_create_archive_with_source_directories_glob_expands(): flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo*'], repository='repo', + location_config={ + 'source_directories': ['foo*'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', ) @@ -225,11 +252,14 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([]) module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo*'], repository='repo', + location_config={ + 'source_directories': ['foo*'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', ) @@ -242,11 +272,14 @@ def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( - exclude_patterns=None, verbosity=None, - storage_config={}, - source_directories=['foo*'], repository='repo', + location_config={ + 'source_directories': ['foo*'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, command='borg', ) From 588955a467c0bbb61692da13f0cd37f7efc93ecc Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 23:27:21 -0700 Subject: [PATCH 152/189] Setting release version. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 4473170..8dc4016 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.1.0.dev0 +1.1.0 * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade. * Added generate-borgmatic-config command for initial config creation. diff --git a/setup.py b/setup.py index 8f8da5c..a8a8678 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.0.dev0' +VERSION = '1.1.0' setup( From b61b09f55cc07f96ab8f92a2ed8d4c3287552607 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Jul 2017 23:27:26 -0700 Subject: [PATCH 153/189] Added tag 1.1.0 for changeset 5a003056a8ff --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 06c448a..1c2351c 100644 --- a/.hgtags +++ b/.hgtags @@ -30,3 +30,4 @@ dbc96d3f83bd5570b6826537616d4160b3374836 0.1.8 de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1 9603d13910b32d57a887765cab694ac5d0acc1f4 1.0.2 32c6341dda9fad77a3982641bce8a3a45821842e 1.0.3 +5a003056a8ff4709c5bd4d6d33354199423f8a1d 1.1.0 From f44a7884e60178e050199f314ad107a9874b58a4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Jul 2017 17:34:17 -0700 Subject: [PATCH 154/189] No longer producing univeral (Python 2 + 3) wheel. --- NEWS | 4 ++++ setup.cfg | 3 --- setup.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 8dc4016..eb86386 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.1.1.dev0 + + * + 1.1.0 * Switched config file format to YAML. Run upgrade-borgmatic-config to upgrade. diff --git a/setup.cfg b/setup.cfg index 4dd5142..12871ff 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,2 @@ [metadata] description-file=README.md - -[bdist_wheel] -universal=1 diff --git a/setup.py b/setup.py index a8a8678..b89ce26 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.0' +VERSION = '1.1.1.dev0' setup( From b36b923c5d6b6860000e5b6bd42392b66307dc84 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 24 Jul 2017 08:41:02 -0700 Subject: [PATCH 155/189] #32: Fix for upgrade-borgmatic-config converting check_last option as a string instead of an integer. --- NEWS | 6 ++++-- borgmatic/config/convert.py | 14 ++++++++++---- borgmatic/tests/unit/config/test_convert.py | 10 ++++++++++ setup.py | 2 +- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/NEWS b/NEWS index eb86386..a580bdc 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ -1.1.1.dev0 +1.1.1 - * + * #32: Fix for upgrade-borgmatic-config converting check_last option as a string instead of an + integer. + * Fix for upgrade-borgmatic-config erroring when consistency checks option is not present. 1.1.0 diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index 832d7d3..923ec48 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -10,10 +10,16 @@ def _convert_section(source_section_config, section_schema): Given a legacy Parsed_config instance for a single section, convert it to its corresponding yaml.comments.CommentedMap representation in preparation for actual serialization to YAML. - Additionally, use the section schema as a source of helpful comments to include within the - returned CommentedMap. + Where integer types exist in the given section schema, convert their values to integers. ''' - destination_section_config = yaml.comments.CommentedMap(source_section_config) + destination_section_config = yaml.comments.CommentedMap([ + ( + option_name, + int(option_value) + if section_schema['map'].get(option_name, {}).get('type') == 'int' else option_value + ) + for option_name, option_value in source_section_config.items() + ]) return destination_section_config @@ -39,7 +45,7 @@ def convert_legacy_parsed_config(source_config, source_excludes, schema): location['repositories'] = [location.pop('repository')] location['exclude_patterns'] = source_excludes - if source_config.consistency['checks']: + if source_config.consistency.get('checks'): destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ') # Add comments to each section, and then add comments to the fields in each section. diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py index 827ecdb..58693ab 100644 --- a/borgmatic/tests/unit/config/test_convert.py +++ b/borgmatic/tests/unit/config/test_convert.py @@ -10,6 +10,16 @@ from borgmatic.config import convert as module Parsed_config = namedtuple('Parsed_config', ('location', 'storage', 'retention', 'consistency')) +def test_convert_section_generates_integer_value_for_integer_type_in_schema(): + flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) + source_section_config = OrderedDict([('check_last', '3')]) + section_schema = {'map': {'check_last': {'type': 'int'}}} + + destination_config = module._convert_section(source_section_config, section_schema) + + assert destination_config == OrderedDict([('check_last', 3)]) + + def test_convert_legacy_parsed_config_transforms_source_config_to_mapping(): flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) source_config = Parsed_config( diff --git a/setup.py b/setup.py index b89ce26..9796fc4 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.1.dev0' +VERSION = '1.1.1' setup( From bcd8b9982d307e9ce3671b37e7320064239cec53 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 24 Jul 2017 08:41:05 -0700 Subject: [PATCH 156/189] Added tag 1.1.1 for changeset 7d3d11eff6c0 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 1c2351c..9273af0 100644 --- a/.hgtags +++ b/.hgtags @@ -31,3 +31,4 @@ de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1 9603d13910b32d57a887765cab694ac5d0acc1f4 1.0.2 32c6341dda9fad77a3982641bce8a3a45821842e 1.0.3 5a003056a8ff4709c5bd4d6d33354199423f8a1d 1.1.0 +7d3d11eff6c0773883c48f221431f157bc7995eb 1.1.1 From 2c61c0bc088d3f9c630a23d71751f463694b7b02 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 24 Jul 2017 19:29:26 -0700 Subject: [PATCH 157/189] #32: Fix for passing check_last as integer to subprocess when calling Borg. --- NEWS | 8 ++++++-- borgmatic/borg.py | 5 ++--- borgmatic/tests/unit/test_borg.py | 4 ++-- setup.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/NEWS b/NEWS index a580bdc..a4b6c8a 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,11 @@ +1.1.2 + + * #32: Fix for passing check_last as integer to subprocess when calling Borg. + 1.1.1 - * #32: Fix for upgrade-borgmatic-config converting check_last option as a string instead of an - integer. + * Part of #32: Fix for upgrade-borgmatic-config converting check_last option as a string instead of + an integer. * Fix for upgrade-borgmatic-config erroring when consistency checks option is not present. 1.1.0 diff --git a/borgmatic/borg.py b/borgmatic/borg.py index f42353d..8c4c730 100644 --- a/borgmatic/borg.py +++ b/borgmatic/borg.py @@ -162,10 +162,9 @@ def _make_check_flags(checks, check_last=None): ('--repository-only',) - Additionally, if a check_last value is given, a "--last" flag will be added. Note that only - Borg supports this flag. + Additionally, if a check_last value is given, a "--last" flag will be added. ''' - last_flag = ('--last', check_last) if check_last else () + last_flag = ('--last', str(check_last)) if check_last else () if checks == DEFAULT_CHECKS: return last_flag diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 36cc312..4963ab5 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -409,13 +409,13 @@ def test_make_check_flags_with_default_checks_returns_no_flags(): def test_make_check_flags_with_checks_and_last_returns_flags_including_last(): flags = module._make_check_flags(('foo', 'bar'), check_last=3) - assert flags == ('--foo-only', '--bar-only', '--last', 3) + assert flags == ('--foo-only', '--bar-only', '--last', '3') def test_make_check_flags_with_last_returns_last_flag(): flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3) - assert flags == ('--last', 3) + assert flags == ('--last', '3') def test_check_archives_should_call_borg_with_parameters(): diff --git a/setup.py b/setup.py index 9796fc4..6d4cc7c 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.1' +VERSION = '1.1.2' setup( From 89cd879529263656aa0ee6119b4caa69f22de923 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 24 Jul 2017 19:29:28 -0700 Subject: [PATCH 158/189] Added tag 1.1.2 for changeset f052a77a8ad5 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 9273af0..24e6798 100644 --- a/.hgtags +++ b/.hgtags @@ -32,3 +32,4 @@ de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1 32c6341dda9fad77a3982641bce8a3a45821842e 1.0.3 5a003056a8ff4709c5bd4d6d33354199423f8a1d 1.1.0 7d3d11eff6c0773883c48f221431f157bc7995eb 1.1.1 +f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2 From e3e4aeff9483f709e4513cf759ab865d60d86f93 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 25 Jul 2017 20:32:32 -0700 Subject: [PATCH 159/189] Fix for generate-borgmatic-config writing config with invalid one_file_system value. --- borgmatic/config/schema.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 64fed07..07e245d 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -6,10 +6,10 @@ map: Where to look for files to backup, and where to store those backups. See https://borgbackup.readthedocs.io/en/stable/quickstart.html and https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details. - required: True + required: true map: source_directories: - required: True + required: true seq: - type: scalar desc: List of source directories to backup (required). Globs are expanded. @@ -20,13 +20,13 @@ map: one_file_system: type: bool desc: Stay in same file system (do not cross mount points). - example: yes + example: true remote_path: type: scalar desc: Alternate Borg remote executable. Defaults to "borg". example: borg1 repositories: - required: True + required: true seq: - type: scalar desc: | @@ -112,7 +112,7 @@ map: seq: - type: str enum: ['repository', 'archives', 'disabled'] - unique: True + unique: true desc: | List of consistency checks to run: "repository", "archives", or both. Defaults to both. Set to "disabled" to disable all consistency checks. See From 0c8816e6cce58725118743a00971c3a80e1ba437 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 25 Jul 2017 21:18:51 -0700 Subject: [PATCH 160/189] #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run. --- NEWS | 5 ++ README.md | 23 +++++++- borgmatic/commands/borgmatic.py | 53 +++++++++-------- borgmatic/config/collect.py | 27 +++++++++ borgmatic/config/convert.py | 11 +++- .../integration/commands/test_borgmatic.py | 15 +++-- borgmatic/tests/unit/config/test_collect.py | 58 +++++++++++++++++++ borgmatic/tests/unit/config/test_convert.py | 16 +++-- setup.py | 2 +- 9 files changed, 172 insertions(+), 38 deletions(-) create mode 100644 borgmatic/config/collect.py create mode 100644 borgmatic/tests/unit/config/test_collect.py diff --git a/NEWS b/NEWS index a4b6c8a..3bc8885 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.1.3.dev0 + + * #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run. + * Fix for generate-borgmatic-config writing config with invalid one_file_system value. + 1.1.2 * #32: Fix for passing check_last as integer to subprocess when calling Borg. diff --git a/README.md b/README.md index 0bd5cda..8ea9a89 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,9 @@ To install borgmatic, run the following command to download and install it: Make sure you're using Python 3, as borgmatic does not support Python 2. (You may have to use "pip3" or similar instead of "pip".) -Then, generate a sample configuration file: +## Configuration + +After you install borgmatic, generate a sample configuration file: sudo generate-borgmatic-config @@ -78,6 +80,25 @@ representative. All fields are optional except where indicated, so feel free to remove anything you don't need. +### Multiple configuration files + +A more advanced usage is to create multiple separate configuration files and +place each one in a /etc/borgmatic.d directory. For instance: + + sudo mkdir /etc/borgmatic.d + sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml + sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml + +With this approach, you can have entirely different backup policies for +different applications on your system. For instance, you may want one backup +configuration for your database data directory, and a different configuration +for your user home directories. + +When you set up multiple configuration files like this, borgmatic will run +each one in turn from a single borgmatic invocation. This includes, by +default, the traditional /etc/borgmatic/config.yaml as well. + + ## Upgrading In general, all you should need to do to upgrade borgmatic is run the diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index a5f39a0..a26c30c 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -5,12 +5,12 @@ from subprocess import CalledProcessError import sys from borgmatic import borg -from borgmatic.config import convert, validate +from borgmatic.config import collect, convert, validate -LEGACY_CONFIG_FILENAME = '/etc/borgmatic/config' -DEFAULT_CONFIG_FILENAME = '/etc/borgmatic/config.yaml' -DEFAULT_EXCLUDES_FILENAME = '/etc/borgmatic/excludes' +LEGACY_CONFIG_PATH = '/etc/borgmatic/config' +DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d'] +DEFAULT_EXCLUDES_PATH = '/etc/borgmatic/excludes' def parse_arguments(*arguments): @@ -21,9 +21,10 @@ def parse_arguments(*arguments): parser = ArgumentParser() parser.add_argument( '-c', '--config', - dest='config_filename', - default=DEFAULT_CONFIG_FILENAME, - help='Configuration filename', + nargs='+', + dest='config_paths', + default=DEFAULT_CONFIG_PATHS, + help='Configuration filenames or directories, defaults to: {}'.format(' '.join(DEFAULT_CONFIG_PATHS)), ) parser.add_argument( '--excludes', @@ -42,25 +43,31 @@ def parse_arguments(*arguments): def main(): # pragma: no cover try: args = parse_arguments(*sys.argv[1:]) - convert.guard_configuration_upgraded(LEGACY_CONFIG_FILENAME, args.config_filename) - config = validate.parse_configuration(args.config_filename, validate.schema_filename()) - (location, storage, retention, consistency) = ( - config.get(section_name, {}) - for section_name in ('location', 'storage', 'retention', 'consistency') - ) - remote_path = location.get('remote_path') + config_filenames = tuple(collect.collect_config_filenames(args.config_paths)) + convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames) - borg.initialize(storage) + if len(config_filenames) == 0: + raise ValueError('Error: No configuration files found in: {}'.format(' '.join(args.config_paths))) - for repository in location['repositories']: - borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) - borg.create_archive( - args.verbosity, - repository, - location, - storage, + for config_filename in config_filenames: + config = validate.parse_configuration(config_filename, validate.schema_filename()) + (location, storage, retention, consistency) = ( + config.get(section_name, {}) + for section_name in ('location', 'storage', 'retention', 'consistency') ) - borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) + remote_path = location.get('remote_path') + + borg.initialize(storage) + + for repository in location['repositories']: + borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) + borg.create_archive( + args.verbosity, + repository, + location, + storage, + ) + borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/borgmatic/config/collect.py b/borgmatic/config/collect.py new file mode 100644 index 0000000..dbe6f2f --- /dev/null +++ b/borgmatic/config/collect.py @@ -0,0 +1,27 @@ +import os + + +def collect_config_filenames(config_paths): + ''' + Given a sequence of config paths, both filenames and directories, resolve that to just an + iterable of files. Accomplish this by listing any given directories looking for contained config + files. This is non-recursive, so any directories within the given directories are ignored. + + Return paths even if they don't exist on disk, so the user can find out about missing + configuration paths. However, skip /etc/borgmatic.d if it's missing, so the user doesn't have to + create it unless they need it. + ''' + for path in config_paths: + exists = os.path.exists(path) + + if os.path.realpath(path) == '/etc/borgmatic.d' and not exists: + continue + + if not os.path.isdir(path) or not exists: + yield path + continue + + for filename in os.listdir(path): + full_filename = os.path.join(path, filename) + if not os.path.isdir(full_filename): + yield full_filename diff --git a/borgmatic/config/convert.py b/borgmatic/config/convert.py index 923ec48..97e7571 100644 --- a/borgmatic/config/convert.py +++ b/borgmatic/config/convert.py @@ -77,14 +77,19 @@ instead of the old one.''' ) -def guard_configuration_upgraded(source_config_filename, destination_config_filename): +def guard_configuration_upgraded(source_config_filename, destination_config_filenames): ''' - If legacy souce configuration exists but destination upgraded config doesn't, raise + If legacy source configuration exists but no destination upgraded configs do, raise LegacyConfigurationNotUpgraded. The idea is that we want to alert the user about upgrading their config if they haven't already. ''' - if os.path.exists(source_config_filename) and not os.path.exists(destination_config_filename): + destination_config_exists = any( + os.path.exists(filename) + for filename in destination_config_filenames + ) + + if os.path.exists(source_config_filename) and not destination_config_exists: raise LegacyConfigurationNotUpgraded() diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py index f1cc43a..0334341 100644 --- a/borgmatic/tests/integration/commands/test_borgmatic.py +++ b/borgmatic/tests/integration/commands/test_borgmatic.py @@ -9,23 +9,30 @@ from borgmatic.commands import borgmatic as module def test_parse_arguments_with_no_arguments_uses_defaults(): parser = module.parse_arguments() - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME + assert parser.config_paths == module.DEFAULT_CONFIG_PATHS assert parser.excludes_filename == None assert parser.verbosity is None -def test_parse_arguments_with_filename_arguments_overrides_defaults(): +def test_parse_arguments_with_path_arguments_overrides_defaults(): parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes') - assert parser.config_filename == 'myconfig' + assert parser.config_paths == ['myconfig'] assert parser.excludes_filename == 'myexcludes' assert parser.verbosity is None +def test_parse_arguments_with_multiple_config_paths_parses_as_list(): + parser = module.parse_arguments('--config', 'myconfig', 'otherconfig') + + assert parser.config_paths == ['myconfig', 'otherconfig'] + assert parser.verbosity is None + + def test_parse_arguments_with_verbosity_flag_overrides_default(): parser = module.parse_arguments('--verbosity', '1') - assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME + assert parser.config_paths == module.DEFAULT_CONFIG_PATHS assert parser.excludes_filename == None assert parser.verbosity == 1 diff --git a/borgmatic/tests/unit/config/test_collect.py b/borgmatic/tests/unit/config/test_collect.py new file mode 100644 index 0000000..2adee63 --- /dev/null +++ b/borgmatic/tests/unit/config/test_collect.py @@ -0,0 +1,58 @@ +from flexmock import flexmock + +from borgmatic.config import collect as module + + +def test_collect_config_filenames_collects_given_files(): + config_paths = ('config.yaml', 'other.yaml') + flexmock(module.os.path).should_receive('isdir').and_return(False) + + config_filenames = tuple(module.collect_config_filenames(config_paths)) + + assert config_filenames == config_paths + + +def test_collect_config_filenames_collects_files_from_given_directories_and_ignores_sub_directories(): + config_paths = ('config.yaml', '/etc/borgmatic.d') + mock_path = flexmock(module.os.path) + mock_path.should_receive('exists').and_return(True) + mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) + mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) + mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False) + mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True) + mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False) + flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar', 'baz.yaml']) + + config_filenames = tuple(module.collect_config_filenames(config_paths)) + + assert config_filenames == ( + 'config.yaml', + '/etc/borgmatic.d/foo.yaml', + '/etc/borgmatic.d/baz.yaml', + ) + + +def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist(): + config_paths = ('config.yaml', '/etc/borgmatic.d') + mock_path = flexmock(module.os.path) + mock_path.should_receive('exists').with_args('config.yaml').and_return(True) + mock_path.should_receive('exists').with_args('/etc/borgmatic.d').and_return(False) + mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) + mock_path.should_receive('isdir').with_args('/etc/borgmatic.d').and_return(True) + + config_filenames = tuple(module.collect_config_filenames(config_paths)) + + assert config_filenames == ('config.yaml',) + + +def test_collect_config_filenames_includes_directory_if_it_does_not_exist(): + config_paths = ('config.yaml', '/my/directory') + mock_path = flexmock(module.os.path) + mock_path.should_receive('exists').with_args('config.yaml').and_return(True) + mock_path.should_receive('exists').with_args('/my/directory').and_return(False) + mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) + mock_path.should_receive('isdir').with_args('/my/directory').and_return(True) + + config_filenames = tuple(module.collect_config_filenames(config_paths)) + + assert config_filenames == config_paths diff --git a/borgmatic/tests/unit/config/test_convert.py b/borgmatic/tests/unit/config/test_convert.py index 58693ab..995fc8a 100644 --- a/borgmatic/tests/unit/config/test_convert.py +++ b/borgmatic/tests/unit/config/test_convert.py @@ -79,30 +79,34 @@ def test_convert_legacy_parsed_config_splits_space_separated_values(): def test_guard_configuration_upgraded_raises_when_only_source_config_present(): flexmock(os.path).should_receive('exists').with_args('config').and_return(True) flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) + flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False) with pytest.raises(module.LegacyConfigurationNotUpgraded): - module.guard_configuration_upgraded('config', 'config.yaml') + module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) def test_guard_configuration_upgraded_does_not_raise_when_only_destination_config_present(): flexmock(os.path).should_receive('exists').with_args('config').and_return(False) - flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True) + flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) + flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True) - module.guard_configuration_upgraded('config', 'config.yaml') + module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) def test_guard_configuration_upgraded_does_not_raise_when_both_configs_present(): flexmock(os.path).should_receive('exists').with_args('config').and_return(True) - flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(True) + flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) + flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(True) - module.guard_configuration_upgraded('config', 'config.yaml') + module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) def test_guard_configuration_upgraded_does_not_raise_when_neither_config_present(): flexmock(os.path).should_receive('exists').with_args('config').and_return(False) flexmock(os.path).should_receive('exists').with_args('config.yaml').and_return(False) + flexmock(os.path).should_receive('exists').with_args('other.yaml').and_return(False) - module.guard_configuration_upgraded('config', 'config.yaml') + module.guard_configuration_upgraded('config', ('config.yaml', 'other.yaml')) def test_guard_excludes_filename_omitted_raises_when_filename_provided(): diff --git a/setup.py b/setup.py index 6d4cc7c..143736b 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.2' +VERSION = '1.1.3.dev0' setup( From 94aaf4554fca77d4151e069065086108112f8145 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 25 Jul 2017 21:21:47 -0700 Subject: [PATCH 161/189] Releasing. --- NEWS | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 3bc8885..f73b4b8 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.1.3.dev0 +1.1.3 * #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run. * Fix for generate-borgmatic-config writing config with invalid one_file_system value. diff --git a/setup.py b/setup.py index 143736b..7831a3e 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.3.dev0' +VERSION = '1.1.3' setup( From 10404143c66c02e3bf8351315de0345c20f66332 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 25 Jul 2017 21:21:50 -0700 Subject: [PATCH 162/189] Added tag 1.1.3 for changeset 3f838f661546 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 24e6798..95e8f52 100644 --- a/.hgtags +++ b/.hgtags @@ -33,3 +33,4 @@ de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1 5a003056a8ff4709c5bd4d6d33354199423f8a1d 1.1.0 7d3d11eff6c0773883c48f221431f157bc7995eb 1.1.1 f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2 +3f838f661546e04529b453aa443529b432afc243 1.1.3 From a2e8abc53794e162bf338629e955227bdf9fe6cc Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Jul 2017 22:02:18 -0700 Subject: [PATCH 163/189] #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or checking enabled. --- NEWS | 6 ++ README.md | 22 ++++++- borgmatic/commands/borgmatic.py | 60 +++++++++++++++---- .../integration/commands/test_borgmatic.py | 24 ++++++++ setup.py | 2 +- 5 files changed, 99 insertions(+), 15 deletions(-) diff --git a/NEWS b/NEWS index f73b4b8..c5a8162 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,9 @@ +1.1.4 + + * #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or + checking enabled. This supports use cases like running consistency checks from a different cron + job with a different frequency, or running pruning with a different verbosity level. + 1.1.3 * #14: Support for running multiple config files in /etc/borgmatic.d/ from a single borgmatic run. diff --git a/README.md b/README.md index 8ea9a89..c15e826 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ to remove anything you don't need. ### Multiple configuration files A more advanced usage is to create multiple separate configuration files and -place each one in a /etc/borgmatic.d directory. For instance: +place each one in an /etc/borgmatic.d directory. For instance: sudo mkdir /etc/borgmatic.d sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml @@ -175,6 +175,12 @@ arguments: This will also prune any old backups as per the configured retention policy, and check backups for consistency problems due to things like file damage. +If you'd like to see the available command-line arguments, view the help: + + borgmatic --help + +### Verbosity + By default, the backup will proceed silently except in the case of errors. But if you'd like to to get additional information about the progress of the backup as it proceeds, use the verbosity option: @@ -185,9 +191,19 @@ Or, for even more progress spew: borgmatic --verbosity 2 -If you'd like to see the available command-line arguments, view the help: +### À la carte - borgmatic --help +If you want to run borgmatic with only pruning, creating, or checking enabled, +the following optional flags are available: + + borgmatic --prune + borgmatic --create + borgmatic --check + +You can run with only one of these flags provided, or you can mix and match +any number of them. This supports use cases like running consistency checks +from a different cron job with a different frequency, or running pruning with +a different verbosity level. ## Autopilot diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index a26c30c..c5c7a28 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -18,7 +18,14 @@ def parse_arguments(*arguments): Given command-line arguments with which this script was invoked, parse the arguments and return them as an ArgumentParser instance. ''' - parser = ArgumentParser() + parser = ArgumentParser( + description= + ''' + A simple wrapper script for the Borg backup software that creates and prunes backups. + If none of the --prune, --create, or --check options are given, then borgmatic defaults + to all three: prune, create, and check archives. + ''' + ) parser.add_argument( '-c', '--config', nargs='+', @@ -29,7 +36,25 @@ def parse_arguments(*arguments): parser.add_argument( '--excludes', dest='excludes_filename', - help='Excludes filename, deprecated in favor of exclude_patterns within configuration', + help='Deprecated in favor of exclude_patterns within configuration', + ) + parser.add_argument( + '-p', '--prune', + dest='prune', + action='store_true', + help='Prune archives according to the retention policy', + ) + parser.add_argument( + '-C', '--create', + dest='create', + action='store_true', + help='Create archives (actually perform backups)', + ) + parser.add_argument( + '-k', '--check', + dest='check', + action='store_true', + help='Check archives for consistency', ) parser.add_argument( '-v', '--verbosity', @@ -37,7 +62,17 @@ def parse_arguments(*arguments): help='Display verbose progress (1 for some, 2 for lots)', ) - return parser.parse_args(arguments) + args = parser.parse_args(arguments) + + # If any of the three action flags in the given parse arguments have been explicitly requested, + # leave them as-is. Otherwise, assume defaults: Mutate the given arguments to enable all the + # actions. + if not args.prune and not args.create and not args.check: + args.prune = True + args.create = True + args.check = True + + return args def main(): # pragma: no cover @@ -60,14 +95,17 @@ def main(): # pragma: no cover borg.initialize(storage) for repository in location['repositories']: - borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) - borg.create_archive( - args.verbosity, - repository, - location, - storage, - ) - borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) + if args.prune: + borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) + if args.create: + borg.create_archive( + args.verbosity, + repository, + location, + storage, + ) + if args.check: + borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py index 0334341..2b82ecf 100644 --- a/borgmatic/tests/integration/commands/test_borgmatic.py +++ b/borgmatic/tests/integration/commands/test_borgmatic.py @@ -37,6 +37,30 @@ def test_parse_arguments_with_verbosity_flag_overrides_default(): assert parser.verbosity == 1 +def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled(): + parser = module.parse_arguments() + + assert parser.prune is True + assert parser.create is True + assert parser.check is True + + +def test_parse_arguments_with_prune_action_leaves_other_actions_disabled(): + parser = module.parse_arguments('--prune') + + assert parser.prune is True + assert parser.create is False + assert parser.check is False + + +def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled(): + parser = module.parse_arguments('--create', '--check') + + assert parser.prune is False + assert parser.create is True + assert parser.check is True + + def test_parse_arguments_with_invalid_arguments_exits(): with pytest.raises(SystemExit): module.parse_arguments('--posix-me-harder') diff --git a/setup.py b/setup.py index 7831a3e..d4f587b 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.3' +VERSION = '1.1.4' setup( From ae15e0f4047180bbbde347def932ee96fa2b51fb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Jul 2017 22:02:43 -0700 Subject: [PATCH 164/189] Added tag 1.1.4 for changeset 3d605962d891 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 95e8f52..50e938a 100644 --- a/.hgtags +++ b/.hgtags @@ -34,3 +34,4 @@ de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1 7d3d11eff6c0773883c48f221431f157bc7995eb 1.1.1 f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2 3f838f661546e04529b453aa443529b432afc243 1.1.3 +3d605962d891731a0f372b903b556ac7a8c8359f 1.1.4 From 0f44fbedf41301fee347413a223aed1547a70288 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Jul 2017 22:36:16 -0700 Subject: [PATCH 165/189] Getting logo to show up on GitHub. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c15e826..09c19a1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ title: borgmatic -borgmatic logo + ## Overview From 23679a6eddec21e7e0e6eb2c4a5e9bf6ed3d513b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 29 Jul 2017 16:05:11 -0700 Subject: [PATCH 166/189] Removing Pelican-specific title metadata out of README markdown. --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 09c19a1..22b1da7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -title: borgmatic - ## Overview From 9bea7ae5edac546ccb014141b1e30f4fe7cb383d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 30 Jul 2017 11:16:26 -0700 Subject: [PATCH 167/189] #34: New "extract" consistency check that performs a dry-run extraction of the most recent archive. --- NEWS | 5 + README.md | 6 ++ borgmatic/borg.py | 66 ++++++++++--- borgmatic/config/schema.yaml | 16 ++-- borgmatic/tests/unit/test_borg.py | 148 +++++++++++++++++++++++++++--- setup.py | 2 +- 6 files changed, 211 insertions(+), 32 deletions(-) diff --git a/NEWS b/NEWS index c5a8162..6222fea 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +1.1.5 + + * #34: New "extract" consistency check that performs a dry-run extraction of the most recent + archive. + 1.1.4 * #17: Added command-line flags for performing a borgmatic run with only pruning, creating, or diff --git a/README.md b/README.md index 22b1da7..057388b 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,12 @@ default). You should edit the file to suit your needs, as the values are just representative. All fields are optional except where indicated, so feel free to remove anything you don't need. +You can also have a look at the [full configuration +schema](https://torsion.org/hg/borgmatic/file/tip/borgmatic/config/schema.yaml) +for the authoritative set of all configuration options. This is handy if +borgmatic has added new options since you originally created your +configuration file. + ### Multiple configuration files diff --git a/borgmatic/borg.py b/borgmatic/borg.py index 8c4c730..fff6e7e 100644 --- a/borgmatic/borg.py +++ b/borgmatic/borg.py @@ -3,6 +3,7 @@ import glob import itertools import os import platform +import sys import re import subprocess import tempfile @@ -43,7 +44,7 @@ def create_archive( ): ''' Given a vebosity flag, a storage config dict, a list of source directories, a local or remote - repository path, a list of exclude patterns, and a command to run, create an attic archive. + repository path, a list of exclude patterns, and a command to run, create a Borg archive. ''' sources = tuple( itertools.chain.from_iterable( @@ -68,8 +69,8 @@ def create_archive( full_command = ( command, 'create', - '{repo}::{hostname}-{timestamp}'.format( - repo=repository, + '{repository}::{hostname}-{timestamp}'.format( + repository=repository, hostname=platform.node(), timestamp=datetime.now().isoformat(), ), @@ -104,7 +105,7 @@ def _make_prune_flags(retention_config): def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None): ''' Given a verbosity flag, a local or remote repository path, a retention config dict, and a - command to run, prune attic archives according the the retention policy specified in that + command to run, prune Borg archives according the the retention policy specified in that configuration. ''' remote_path_flags = ('--remote-path', remote_path) if remote_path else () @@ -170,33 +171,72 @@ def _make_check_flags(checks, check_last=None): return tuple( '--{}-only'.format(check) for check in checks + if check in DEFAULT_CHECKS ) + last_flag def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None): ''' Given a verbosity flag, a local or remote repository path, a consistency config dict, and a - command to run, check the contained attic archives for consistency. + command to run, check the contained Borg archives for consistency. If there are no consistency checks to run, skip running them. ''' checks = _parse_checks(consistency_config) check_last = consistency_config.get('check_last', None) - if not checks: - return + if set(checks).intersection(set(DEFAULT_CHECKS)): + remote_path_flags = ('--remote-path', remote_path) if remote_path else () + verbosity_flags = { + VERBOSITY_SOME: ('--info',), + VERBOSITY_LOTS: ('--debug',), + }.get(verbosity, ()) + + full_command = ( + command, 'check', + repository, + ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags + + # The check command spews to stdout/stderr even without the verbose flag. Suppress it. + stdout = None if verbosity_flags else open(os.devnull, 'w') + + subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) + + if 'extract' in checks: + extract_last_archive_dry_run(verbosity, repository, command, remote_path) + + +def extract_last_archive_dry_run(verbosity, repository, command=COMMAND, remote_path=None): + ''' + Perform an extraction dry-run of just the most recent archive. If there are no archives, skip + the dry-run. + ''' remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { VERBOSITY_SOME: ('--info',), VERBOSITY_LOTS: ('--debug',), }.get(verbosity, ()) - full_command = ( - command, 'check', + full_list_command = ( + command, 'list', + '--short', repository, - ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags + ) + remote_path_flags + verbosity_flags - # The check command spews to stdout/stderr even without the verbose flag. Suppress it. - stdout = None if verbosity_flags else open(os.devnull, 'w') + list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding) - subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) + last_archive_name = list_output.strip().split('\n')[-1] + if not last_archive_name: + return + + list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else () + full_extract_command = ( + command, 'extract', + '--dry-run', + '{repository}::{last_archive_name}'.format( + repository=repository, + last_archive_name=last_archive_name, + ), + ) + remote_path_flags + verbosity_flags + list_flag + + subprocess.check_call(full_extract_command) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 07e245d..567ec73 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -106,21 +106,25 @@ map: consistency: desc: | Consistency checks to run after backups. See - https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and + https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details. map: checks: seq: - type: str - enum: ['repository', 'archives', 'disabled'] + enum: ['repository', 'archives', 'extract', 'disabled'] unique: true desc: | - List of consistency checks to run: "repository", "archives", or both. Defaults - to both. Set to "disabled" to disable all consistency checks. See - https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details. + List of one or more consistency checks to run: "repository", "archives", and/or + "extract". Defaults to "repository" and "archives". Set to "disabled" to disable + all consistency checks. "repository" checks the consistency of the repository, + "archive" checks all of the archives, and "extract" does an extraction dry-run + of just the most recent archive. example: - repository - archives check_last: type: int - desc: Restrict the number of checked archives to the last n. + desc: Restrict the number of checked archives to the last n. Applies only to the + "archives" check. example: 3 diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py index 4963ab5..1cba040 100644 --- a/borgmatic/tests/unit/test_borg.py +++ b/borgmatic/tests/unit/test_borg.py @@ -4,6 +4,7 @@ import sys import os from flexmock import flexmock +import pytest from borgmatic import borg as module from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS @@ -46,17 +47,23 @@ def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise(): def insert_subprocess_mock(check_call_command, **kwargs): - subprocess = flexmock(STDOUT=STDOUT) + subprocess = flexmock(module.subprocess) subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() flexmock(module).subprocess = subprocess def insert_subprocess_never(): - subprocess = flexmock() + subprocess = flexmock(module.subprocess) subprocess.should_receive('check_call').never() flexmock(module).subprocess = subprocess +def insert_subprocess_check_output_mock(check_output_command, result, **kwargs): + subprocess = flexmock(module.subprocess) + subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once() + flexmock(module).subprocess = subprocess + + def insert_platform_mock(): flexmock(module.platform).should_receive('node').and_return('host') @@ -395,9 +402,15 @@ def test_parse_checks_with_disabled_returns_no_checks(): def test_make_check_flags_with_checks_returns_flags(): - flags = module._make_check_flags(('foo', 'bar')) + flags = module._make_check_flags(('repository',)) - assert flags == ('--foo-only', '--bar-only') + assert flags == ('--repository-only',) + + +def test_make_check_flags_with_extract_check_does_not_make_extract_flag(): + flags = module._make_check_flags(('extract',)) + + assert flags == () def test_make_check_flags_with_default_checks_returns_no_flags(): @@ -407,19 +420,27 @@ def test_make_check_flags_with_default_checks_returns_no_flags(): def test_make_check_flags_with_checks_and_last_returns_flags_including_last(): - flags = module._make_check_flags(('foo', 'bar'), check_last=3) + flags = module._make_check_flags(('repository',), check_last=3) - assert flags == ('--foo-only', '--bar-only', '--last', '3') + assert flags == ('--repository-only', '--last', '3') -def test_make_check_flags_with_last_returns_last_flag(): +def test_make_check_flags_with_default_checks_and_last_returns_last_flag(): flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3) assert flags == ('--last', '3') -def test_check_archives_should_call_borg_with_parameters(): - checks = flexmock() +@pytest.mark.parametrize( + 'checks', + ( + ('repository',), + ('archives',), + ('repository', 'archives'), + ('repository', 'archives', 'other'), + ), +) +def test_check_archives_should_call_borg_with_parameters(checks): check_last = flexmock() consistency_config = flexmock().should_receive('get').and_return(check_last).mock flexmock(module).should_receive('_parse_checks').and_return(checks) @@ -442,9 +463,27 @@ def test_check_archives_should_call_borg_with_parameters(): ) +def test_check_archives_with_extract_check_should_call_extract_only(): + checks = ('extract',) + check_last = flexmock() + consistency_config = flexmock().should_receive('get').and_return(check_last).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').never() + flexmock(module).should_receive('extract_last_archive_dry_run').once() + insert_subprocess_never() + + module.check_archives( + verbosity=None, + repository='repo', + consistency_config=consistency_config, + command='borg', + ) + + def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter(): + checks = ('repository',) consistency_config = flexmock().should_receive('get').and_return(None).mock - flexmock(module).should_receive('_parse_checks').and_return(flexmock()) + flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( ('borg', 'check', 'repo', '--info'), @@ -462,8 +501,9 @@ def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter(): + checks = ('repository',) consistency_config = flexmock().should_receive('get').and_return(None).mock - flexmock(module).should_receive('_parse_checks').and_return(flexmock()) + flexmock(module).should_receive('_parse_checks').and_return(checks) flexmock(module).should_receive('_make_check_flags').and_return(()) insert_subprocess_mock( ('borg', 'check', 'repo', '--debug'), @@ -494,7 +534,7 @@ def test_check_archives_without_any_checks_should_bail(): def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters(): - checks = flexmock() + checks = ('repository',) check_last = flexmock() consistency_config = flexmock().should_receive('get').and_return(check_last).mock flexmock(module).should_receive('_parse_checks').and_return(checks) @@ -516,3 +556,87 @@ def test_check_archives_with_remote_path_should_call_borg_with_remote_path_param command='borg', remote_path='borg1', ) + + +def test_extract_last_archive_dry_run_should_call_borg_with_last_archive(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2'), + ) + + module.extract_last_archive_dry_run( + verbosity=None, + repository='repo', + command='borg', + ) + + +def test_extract_last_archive_dry_run_without_any_archives_should_bail(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo'), + result='\n'.encode('utf-8'), + ) + insert_subprocess_never() + + module.extract_last_archive_dry_run( + verbosity=None, + repository='repo', + command='borg', + ) + + +def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo', '--info'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2', '--info'), + ) + + module.extract_last_archive_dry_run( + verbosity=VERBOSITY_SOME, + repository='repo', + command='borg', + ) + + +def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo', '--debug'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'), + ) + + module.extract_last_archive_dry_run( + verbosity=VERBOSITY_LOTS, + repository='repo', + command='borg', + ) + + +def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'), + ) + + module.extract_last_archive_dry_run( + verbosity=None, + repository='repo', + command='borg', + remote_path='borg1', + ) diff --git a/setup.py b/setup.py index d4f587b..2ff9690 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.4' +VERSION = '1.1.5' setup( From 77d3c66fb98ab72cd27000b5fcf8e81ae56b8125 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 30 Jul 2017 11:16:41 -0700 Subject: [PATCH 168/189] Added tag 1.1.5 for changeset 64ca13bfe050 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 50e938a..50ee67d 100644 --- a/.hgtags +++ b/.hgtags @@ -35,3 +35,4 @@ de2d7721cdec93a52d20222a9ddd579ed93c1017 1.0.1 f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2 3f838f661546e04529b453aa443529b432afc243 1.1.3 3d605962d891731a0f372b903b556ac7a8c8359f 1.1.4 +64ca13bfe050f656b44ed2eb1c3db045bfddd133 1.1.5 From aa04473521f0ce9c0bb4469cd60daea501a5fb6c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 5 Aug 2017 16:21:39 -0700 Subject: [PATCH 169/189] Split out Borg integration code into multiple files, as it was getting kind of hairy all in one. --- borgmatic/borg.py | 242 -------- borgmatic/borg/__init__.py | 0 borgmatic/borg/check.py | 85 +++ borgmatic/borg/create.py | 72 +++ borgmatic/borg/extract.py | 40 ++ borgmatic/borg/prune.py | 48 ++ borgmatic/tests/unit/borg/__init__.py | 0 borgmatic/tests/unit/borg/test_check.py | 185 +++++++ borgmatic/tests/unit/borg/test_create.py | 264 +++++++++ borgmatic/tests/unit/borg/test_extract.py | 100 ++++ borgmatic/tests/unit/borg/test_prune.py | 93 ++++ borgmatic/tests/unit/test_borg.py | 642 ---------------------- 12 files changed, 887 insertions(+), 884 deletions(-) delete mode 100644 borgmatic/borg.py create mode 100644 borgmatic/borg/__init__.py create mode 100644 borgmatic/borg/check.py create mode 100644 borgmatic/borg/create.py create mode 100644 borgmatic/borg/extract.py create mode 100644 borgmatic/borg/prune.py create mode 100644 borgmatic/tests/unit/borg/__init__.py create mode 100644 borgmatic/tests/unit/borg/test_check.py create mode 100644 borgmatic/tests/unit/borg/test_create.py create mode 100644 borgmatic/tests/unit/borg/test_extract.py create mode 100644 borgmatic/tests/unit/borg/test_prune.py delete mode 100644 borgmatic/tests/unit/test_borg.py diff --git a/borgmatic/borg.py b/borgmatic/borg.py deleted file mode 100644 index fff6e7e..0000000 --- a/borgmatic/borg.py +++ /dev/null @@ -1,242 +0,0 @@ -from datetime import datetime -import glob -import itertools -import os -import platform -import sys -import re -import subprocess -import tempfile - -from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS - - -# Integration with Borg for actually handling backups. - - -COMMAND = 'borg' - - -def initialize(storage_config, command=COMMAND): - passphrase = storage_config.get('encryption_passphrase') - - if passphrase: - os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase - - -def _write_exclude_file(exclude_patterns=None): - ''' - Given a sequence of exclude patterns, write them to a named temporary file and return it. Return - None if no patterns are provided. - ''' - if not exclude_patterns: - return None - - exclude_file = tempfile.NamedTemporaryFile('w') - exclude_file.write('\n'.join(exclude_patterns)) - exclude_file.flush() - - return exclude_file - - -def create_archive( - verbosity, repository, location_config, storage_config, command=COMMAND, -): - ''' - Given a vebosity flag, a storage config dict, a list of source directories, a local or remote - repository path, a list of exclude patterns, and a command to run, create a Borg archive. - ''' - sources = tuple( - itertools.chain.from_iterable( - glob.glob(directory) or [directory] - for directory in location_config['source_directories'] - ) - ) - - exclude_file = _write_exclude_file(location_config.get('exclude_patterns')) - exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else () - compression = storage_config.get('compression', None) - compression_flags = ('--compression', compression) if compression else () - umask = storage_config.get('umask', None) - umask_flags = ('--umask', str(umask)) if umask else () - one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else () - remote_path = location_config.get('remote_path') - remote_path_flags = ('--remote-path', remote_path) if remote_path else () - verbosity_flags = { - VERBOSITY_SOME: ('--info', '--stats',), - VERBOSITY_LOTS: ('--debug', '--list', '--stats'), - }.get(verbosity, ()) - - full_command = ( - command, 'create', - '{repository}::{hostname}-{timestamp}'.format( - repository=repository, - hostname=platform.node(), - timestamp=datetime.now().isoformat(), - ), - ) + sources + exclude_flags + compression_flags + one_file_system_flags + \ - remote_path_flags + umask_flags + verbosity_flags - - subprocess.check_call(full_command) - - -def _make_prune_flags(retention_config): - ''' - Given a retention config dict mapping from option name to value, tranform it into an iterable of - command-line name-value flag pairs. - - For example, given a retention config of: - - {'keep_weekly': 4, 'keep_monthly': 6} - - This will be returned as an iterable of: - - ( - ('--keep-weekly', '4'), - ('--keep-monthly', '6'), - ) - ''' - return ( - ('--' + option_name.replace('_', '-'), str(retention_config[option_name])) - for option_name, value in retention_config.items() - ) - - -def prune_archives(verbosity, repository, retention_config, command=COMMAND, remote_path=None): - ''' - Given a verbosity flag, a local or remote repository path, a retention config dict, and a - command to run, prune Borg archives according the the retention policy specified in that - configuration. - ''' - remote_path_flags = ('--remote-path', remote_path) if remote_path else () - verbosity_flags = { - VERBOSITY_SOME: ('--info', '--stats',), - VERBOSITY_LOTS: ('--debug', '--stats'), - }.get(verbosity, ()) - - full_command = ( - command, 'prune', - repository, - ) + tuple( - element - for pair in _make_prune_flags(retention_config) - for element in pair - ) + remote_path_flags + verbosity_flags - - subprocess.check_call(full_command) - - -DEFAULT_CHECKS = ('repository', 'archives') - - -def _parse_checks(consistency_config): - ''' - Given a consistency config with a "checks" list, transform it to a tuple of named checks to run. - - For example, given a retention config of: - - {'checks': ['repository', 'archives']} - - This will be returned as: - - ('repository', 'archives') - - If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string - "disabled", return an empty tuple, meaning that no checks should be run. - ''' - checks = consistency_config.get('checks', []) - if checks == ['disabled']: - return () - - return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS - - -def _make_check_flags(checks, check_last=None): - ''' - Given a parsed sequence of checks, transform it into tuple of command-line flags. - - For example, given parsed checks of: - - ('repository',) - - This will be returned as: - - ('--repository-only',) - - Additionally, if a check_last value is given, a "--last" flag will be added. - ''' - last_flag = ('--last', str(check_last)) if check_last else () - if checks == DEFAULT_CHECKS: - return last_flag - - return tuple( - '--{}-only'.format(check) for check in checks - if check in DEFAULT_CHECKS - ) + last_flag - - -def check_archives(verbosity, repository, consistency_config, command=COMMAND, remote_path=None): - ''' - Given a verbosity flag, a local or remote repository path, a consistency config dict, and a - command to run, check the contained Borg archives for consistency. - - If there are no consistency checks to run, skip running them. - ''' - checks = _parse_checks(consistency_config) - check_last = consistency_config.get('check_last', None) - - if set(checks).intersection(set(DEFAULT_CHECKS)): - remote_path_flags = ('--remote-path', remote_path) if remote_path else () - verbosity_flags = { - VERBOSITY_SOME: ('--info',), - VERBOSITY_LOTS: ('--debug',), - }.get(verbosity, ()) - - full_command = ( - command, 'check', - repository, - ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags - - # The check command spews to stdout/stderr even without the verbose flag. Suppress it. - stdout = None if verbosity_flags else open(os.devnull, 'w') - - subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) - - if 'extract' in checks: - extract_last_archive_dry_run(verbosity, repository, command, remote_path) - - -def extract_last_archive_dry_run(verbosity, repository, command=COMMAND, remote_path=None): - ''' - Perform an extraction dry-run of just the most recent archive. If there are no archives, skip - the dry-run. - ''' - remote_path_flags = ('--remote-path', remote_path) if remote_path else () - verbosity_flags = { - VERBOSITY_SOME: ('--info',), - VERBOSITY_LOTS: ('--debug',), - }.get(verbosity, ()) - - full_list_command = ( - command, 'list', - '--short', - repository, - ) + remote_path_flags + verbosity_flags - - list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding) - - last_archive_name = list_output.strip().split('\n')[-1] - if not last_archive_name: - return - - list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else () - full_extract_command = ( - command, 'extract', - '--dry-run', - '{repository}::{last_archive_name}'.format( - repository=repository, - last_archive_name=last_archive_name, - ), - ) + remote_path_flags + verbosity_flags + list_flag - - subprocess.check_call(full_extract_command) diff --git a/borgmatic/borg/__init__.py b/borgmatic/borg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py new file mode 100644 index 0000000..4980554 --- /dev/null +++ b/borgmatic/borg/check.py @@ -0,0 +1,85 @@ +import os +import subprocess + +from borgmatic.borg import extract +from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS + + +DEFAULT_CHECKS = ('repository', 'archives') + + +def _parse_checks(consistency_config): + ''' + Given a consistency config with a "checks" list, transform it to a tuple of named checks to run. + + For example, given a retention config of: + + {'checks': ['repository', 'archives']} + + This will be returned as: + + ('repository', 'archives') + + If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string + "disabled", return an empty tuple, meaning that no checks should be run. + ''' + checks = consistency_config.get('checks', []) + if checks == ['disabled']: + return () + + return tuple(check for check in checks if check.lower() not in ('disabled', '')) or DEFAULT_CHECKS + + +def _make_check_flags(checks, check_last=None): + ''' + Given a parsed sequence of checks, transform it into tuple of command-line flags. + + For example, given parsed checks of: + + ('repository',) + + This will be returned as: + + ('--repository-only',) + + Additionally, if a check_last value is given, a "--last" flag will be added. + ''' + last_flag = ('--last', str(check_last)) if check_last else () + if checks == DEFAULT_CHECKS: + return last_flag + + return tuple( + '--{}-only'.format(check) for check in checks + if check in DEFAULT_CHECKS + ) + last_flag + + +def check_archives(verbosity, repository, consistency_config, remote_path=None): + ''' + Given a verbosity flag, a local or remote repository path, a consistency config dict, and a + command to run, check the contained Borg archives for consistency. + + If there are no consistency checks to run, skip running them. + ''' + checks = _parse_checks(consistency_config) + check_last = consistency_config.get('check_last', None) + + if set(checks).intersection(set(DEFAULT_CHECKS)): + remote_path_flags = ('--remote-path', remote_path) if remote_path else () + verbosity_flags = { + VERBOSITY_SOME: ('--info',), + VERBOSITY_LOTS: ('--debug',), + }.get(verbosity, ()) + + full_command = ( + 'borg', 'check', + repository, + ) + _make_check_flags(checks, check_last) + remote_path_flags + verbosity_flags + + # The check command spews to stdout/stderr even without the verbose flag. Suppress it. + stdout = None if verbosity_flags else open(os.devnull, 'w') + + subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT) + + if 'extract' in checks: + extract.extract_last_archive_dry_run(verbosity, repository, remote_path) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py new file mode 100644 index 0000000..21af701 --- /dev/null +++ b/borgmatic/borg/create.py @@ -0,0 +1,72 @@ +from datetime import datetime +import glob +import itertools +import os +import platform +import subprocess +import tempfile + +from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS + + +def initialize(storage_config): + passphrase = storage_config.get('encryption_passphrase') + + if passphrase: + os.environ['BORG_PASSPHRASE'] = passphrase + + +def _write_exclude_file(exclude_patterns=None): + ''' + Given a sequence of exclude patterns, write them to a named temporary file and return it. Return + None if no patterns are provided. + ''' + if not exclude_patterns: + return None + + exclude_file = tempfile.NamedTemporaryFile('w') + exclude_file.write('\n'.join(exclude_patterns)) + exclude_file.flush() + + return exclude_file + + +def create_archive( + verbosity, repository, location_config, storage_config, +): + ''' + Given a vebosity flag, a storage config dict, a list of source directories, a local or remote + repository path, a list of exclude patterns, create a Borg archive. + ''' + sources = tuple( + itertools.chain.from_iterable( + glob.glob(directory) or [directory] + for directory in location_config['source_directories'] + ) + ) + + exclude_file = _write_exclude_file(location_config.get('exclude_patterns')) + exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else () + compression = storage_config.get('compression', None) + compression_flags = ('--compression', compression) if compression else () + umask = storage_config.get('umask', None) + umask_flags = ('--umask', str(umask)) if umask else () + one_file_system_flags = ('--one-file-system',) if location_config.get('one_file_system') else () + remote_path = location_config.get('remote_path') + remote_path_flags = ('--remote-path', remote_path) if remote_path else () + verbosity_flags = { + VERBOSITY_SOME: ('--info', '--stats',), + VERBOSITY_LOTS: ('--debug', '--list', '--stats'), + }.get(verbosity, ()) + + full_command = ( + 'borg', 'create', + '{repository}::{hostname}-{timestamp}'.format( + repository=repository, + hostname=platform.node(), + timestamp=datetime.now().isoformat(), + ), + ) + sources + exclude_flags + compression_flags + one_file_system_flags + \ + remote_path_flags + umask_flags + verbosity_flags + + subprocess.check_call(full_command) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py new file mode 100644 index 0000000..fde1ac5 --- /dev/null +++ b/borgmatic/borg/extract.py @@ -0,0 +1,40 @@ +import sys +import subprocess + +from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS + + +def extract_last_archive_dry_run(verbosity, repository, remote_path=None): + ''' + Perform an extraction dry-run of just the most recent archive. If there are no archives, skip + the dry-run. + ''' + remote_path_flags = ('--remote-path', remote_path) if remote_path else () + verbosity_flags = { + VERBOSITY_SOME: ('--info',), + VERBOSITY_LOTS: ('--debug',), + }.get(verbosity, ()) + + full_list_command = ( + 'borg', 'list', + '--short', + repository, + ) + remote_path_flags + verbosity_flags + + list_output = subprocess.check_output(full_list_command).decode(sys.stdout.encoding) + + last_archive_name = list_output.strip().split('\n')[-1] + if not last_archive_name: + return + + list_flag = ('--list',) if verbosity == VERBOSITY_LOTS else () + full_extract_command = ( + 'borg', 'extract', + '--dry-run', + '{repository}::{last_archive_name}'.format( + repository=repository, + last_archive_name=last_archive_name, + ), + ) + remote_path_flags + verbosity_flags + list_flag + + subprocess.check_call(full_extract_command) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py new file mode 100644 index 0000000..8f52cb4 --- /dev/null +++ b/borgmatic/borg/prune.py @@ -0,0 +1,48 @@ +import subprocess + +from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS + + +def _make_prune_flags(retention_config): + ''' + Given a retention config dict mapping from option name to value, tranform it into an iterable of + command-line name-value flag pairs. + + For example, given a retention config of: + + {'keep_weekly': 4, 'keep_monthly': 6} + + This will be returned as an iterable of: + + ( + ('--keep-weekly', '4'), + ('--keep-monthly', '6'), + ) + ''' + return ( + ('--' + option_name.replace('_', '-'), str(retention_config[option_name])) + for option_name, value in retention_config.items() + ) + + +def prune_archives(verbosity, repository, retention_config, remote_path=None): + ''' + Given a verbosity flag, a local or remote repository path, a retention config dict, prune Borg + archives according the the retention policy specified in that configuration. + ''' + remote_path_flags = ('--remote-path', remote_path) if remote_path else () + verbosity_flags = { + VERBOSITY_SOME: ('--info', '--stats',), + VERBOSITY_LOTS: ('--debug', '--stats'), + }.get(verbosity, ()) + + full_command = ( + 'borg', 'prune', + repository, + ) + tuple( + element + for pair in _make_prune_flags(retention_config) + for element in pair + ) + remote_path_flags + verbosity_flags + + subprocess.check_call(full_command) diff --git a/borgmatic/tests/unit/borg/__init__.py b/borgmatic/tests/unit/borg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/borgmatic/tests/unit/borg/test_check.py b/borgmatic/tests/unit/borg/test_check.py new file mode 100644 index 0000000..2b66f09 --- /dev/null +++ b/borgmatic/tests/unit/borg/test_check.py @@ -0,0 +1,185 @@ +from subprocess import STDOUT +import sys + +from flexmock import flexmock +import pytest + +from borgmatic.borg import check as module +from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS + + +def insert_subprocess_mock(check_call_command, **kwargs): + subprocess = flexmock(module.subprocess) + subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() + + +def insert_subprocess_never(): + subprocess = flexmock(module.subprocess) + subprocess.should_receive('check_call').never() + + +def test_parse_checks_returns_them_as_tuple(): + checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']}) + + assert checks == ('foo', 'bar') + + +def test_parse_checks_with_missing_value_returns_defaults(): + checks = module._parse_checks({}) + + assert checks == module.DEFAULT_CHECKS + + +def test_parse_checks_with_blank_value_returns_defaults(): + checks = module._parse_checks({'checks': []}) + + assert checks == module.DEFAULT_CHECKS + + +def test_parse_checks_with_disabled_returns_no_checks(): + checks = module._parse_checks({'checks': ['disabled']}) + + assert checks == () + + +def test_make_check_flags_with_checks_returns_flags(): + flags = module._make_check_flags(('repository',)) + + assert flags == ('--repository-only',) + + +def test_make_check_flags_with_extract_check_does_not_make_extract_flag(): + flags = module._make_check_flags(('extract',)) + + assert flags == () + + +def test_make_check_flags_with_default_checks_returns_no_flags(): + flags = module._make_check_flags(module.DEFAULT_CHECKS) + + assert flags == () + + +def test_make_check_flags_with_checks_and_last_returns_flags_including_last(): + flags = module._make_check_flags(('repository',), check_last=3) + + assert flags == ('--repository-only', '--last', '3') + + +def test_make_check_flags_with_default_checks_and_last_returns_last_flag(): + flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3) + + assert flags == ('--last', '3') + + +@pytest.mark.parametrize( + 'checks', + ( + ('repository',), + ('archives',), + ('repository', 'archives'), + ('repository', 'archives', 'other'), + ), +) +def test_check_archives_should_call_borg_with_parameters(checks): + check_last = flexmock() + consistency_config = flexmock().should_receive('get').and_return(check_last).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) + stdout = flexmock() + insert_subprocess_mock( + ('borg', 'check', 'repo'), + stdout=stdout, stderr=STDOUT, + ) + flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout) + flexmock(module.os).should_receive('devnull') + + module.check_archives( + verbosity=None, + repository='repo', + consistency_config=consistency_config, + ) + + +def test_check_archives_with_extract_check_should_call_extract_only(): + checks = ('extract',) + check_last = flexmock() + consistency_config = flexmock().should_receive('get').and_return(check_last).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').never() + flexmock(module.extract).should_receive('extract_last_archive_dry_run').once() + insert_subprocess_never() + + module.check_archives( + verbosity=None, + repository='repo', + consistency_config=consistency_config, + ) + + +def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter(): + checks = ('repository',) + consistency_config = flexmock().should_receive('get').and_return(None).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').and_return(()) + insert_subprocess_mock( + ('borg', 'check', 'repo', '--info'), + stdout=None, stderr=STDOUT, + ) + + module.check_archives( + verbosity=VERBOSITY_SOME, + repository='repo', + consistency_config=consistency_config, + ) + + +def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter(): + checks = ('repository',) + consistency_config = flexmock().should_receive('get').and_return(None).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').and_return(()) + insert_subprocess_mock( + ('borg', 'check', 'repo', '--debug'), + stdout=None, stderr=STDOUT, + ) + + module.check_archives( + verbosity=VERBOSITY_LOTS, + repository='repo', + consistency_config=consistency_config, + ) + + +def test_check_archives_without_any_checks_should_bail(): + consistency_config = flexmock().should_receive('get').and_return(None).mock + flexmock(module).should_receive('_parse_checks').and_return(()) + insert_subprocess_never() + + module.check_archives( + verbosity=None, + repository='repo', + consistency_config=consistency_config, + ) + + +def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters(): + checks = ('repository',) + check_last = flexmock() + consistency_config = flexmock().should_receive('get').and_return(check_last).mock + flexmock(module).should_receive('_parse_checks').and_return(checks) + flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) + stdout = flexmock() + insert_subprocess_mock( + ('borg', 'check', 'repo', '--remote-path', 'borg1'), + stdout=stdout, stderr=STDOUT, + ) + flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout) + flexmock(module.os).should_receive('devnull') + + module.check_archives( + verbosity=None, + repository='repo', + consistency_config=consistency_config, + remote_path='borg1', + ) diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py new file mode 100644 index 0000000..3b4d268 --- /dev/null +++ b/borgmatic/tests/unit/borg/test_create.py @@ -0,0 +1,264 @@ +import os + +from flexmock import flexmock + +from borgmatic.borg import create as module +from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS + + +def test_initialize_with_passphrase_should_set_environment(): + orig_environ = os.environ + + try: + os.environ = {} + module.initialize({'encryption_passphrase': 'pass'}) + assert os.environ.get('BORG_PASSPHRASE') == 'pass' + finally: + os.environ = orig_environ + + +def test_initialize_without_passphrase_should_not_set_environment(): + orig_environ = os.environ + + try: + os.environ = {} + module.initialize({}) + assert os.environ.get('BORG_PASSPHRASE') == None + finally: + os.environ = orig_environ + + +def test_write_exclude_file_does_not_raise(): + temporary_file = flexmock( + name='filename', + write=lambda mode: None, + flush=lambda: None, + ) + flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file) + + module._write_exclude_file(['exclude']) + + +def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise(): + module._write_exclude_file([]) + + +def insert_subprocess_mock(check_call_command, **kwargs): + subprocess = flexmock(module.subprocess) + subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() + + +def insert_platform_mock(): + flexmock(module.platform).should_receive('node').and_return('host') + + +def insert_datetime_mock(): + flexmock(module).datetime = flexmock().should_receive('now').and_return( + flexmock().should_receive('isoformat').and_return('now').mock + ).mock + + +CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar') + + +def test_create_archive_should_call_borg_with_parameters(): + flexmock(module).should_receive('_write_exclude_file') + insert_subprocess_mock(CREATE_COMMAND) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, + ) + + +def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): + flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes')) + insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes')) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': ['exclude'], + }, + storage_config={}, + ) + + +def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter(): + flexmock(module).should_receive('_write_exclude_file') + insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',)) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + verbosity=VERBOSITY_SOME, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, + ) + + +def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter(): + flexmock(module).should_receive('_write_exclude_file') + insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats')) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + verbosity=VERBOSITY_LOTS, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, + ) + + +def test_create_archive_with_compression_should_call_borg_with_compression_parameters(): + flexmock(module).should_receive('_write_exclude_file') + insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'compression': 'rle'}, + ) + + +def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters(): + flexmock(module).should_receive('_write_exclude_file') + insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'one_file_system': True, + 'exclude_patterns': None, + }, + storage_config={}, + ) + + +def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): + flexmock(module).should_receive('_write_exclude_file') + insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1')) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'remote_path': 'borg1', + 'exclude_patterns': None, + }, + storage_config={}, + ) + + +def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): + flexmock(module).should_receive('_write_exclude_file') + insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) + insert_platform_mock() + insert_datetime_mock() + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'umask': 740}, + ) + + +def test_create_archive_with_source_directories_glob_expands(): + flexmock(module).should_receive('_write_exclude_file') + insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) + insert_platform_mock() + insert_datetime_mock() + flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo*'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, + ) + + +def test_create_archive_with_non_matching_source_directories_glob_passes_through(): + flexmock(module).should_receive('_write_exclude_file') + insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*')) + insert_platform_mock() + insert_datetime_mock() + flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([]) + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo*'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, + ) + + +def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): + flexmock(module).should_receive('_write_exclude_file') + insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) + insert_platform_mock() + insert_datetime_mock() + flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo*'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={}, + ) diff --git a/borgmatic/tests/unit/borg/test_extract.py b/borgmatic/tests/unit/borg/test_extract.py new file mode 100644 index 0000000..929c324 --- /dev/null +++ b/borgmatic/tests/unit/borg/test_extract.py @@ -0,0 +1,100 @@ +import sys + +from flexmock import flexmock + +from borgmatic.borg import extract as module +from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS + + +def insert_subprocess_mock(check_call_command, **kwargs): + subprocess = flexmock(module.subprocess) + subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() + + +def insert_subprocess_never(): + subprocess = flexmock(module.subprocess) + subprocess.should_receive('check_call').never() + + +def insert_subprocess_check_output_mock(check_output_command, result, **kwargs): + subprocess = flexmock(module.subprocess) + subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once() + + +def test_extract_last_archive_dry_run_should_call_borg_with_last_archive(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2'), + ) + + module.extract_last_archive_dry_run( + verbosity=None, + repository='repo', + ) + + +def test_extract_last_archive_dry_run_without_any_archives_should_bail(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo'), + result='\n'.encode('utf-8'), + ) + insert_subprocess_never() + + module.extract_last_archive_dry_run( + verbosity=None, + repository='repo', + ) + + +def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo', '--info'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2', '--info'), + ) + + module.extract_last_archive_dry_run( + verbosity=VERBOSITY_SOME, + repository='repo', + ) + + +def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo', '--debug'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'), + ) + + module.extract_last_archive_dry_run( + verbosity=VERBOSITY_LOTS, + repository='repo', + ) + + +def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters(): + flexmock(sys.stdout).encoding = 'utf-8' + insert_subprocess_check_output_mock( + ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'), + result='archive1\narchive2\n'.encode('utf-8'), + ) + insert_subprocess_mock( + ('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'), + ) + + module.extract_last_archive_dry_run( + verbosity=None, + repository='repo', + remote_path='borg1', + ) diff --git a/borgmatic/tests/unit/borg/test_prune.py b/borgmatic/tests/unit/borg/test_prune.py new file mode 100644 index 0000000..379960a --- /dev/null +++ b/borgmatic/tests/unit/borg/test_prune.py @@ -0,0 +1,93 @@ +from collections import OrderedDict + +from flexmock import flexmock + +from borgmatic.borg import prune as module +from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS + + +def insert_subprocess_mock(check_call_command, **kwargs): + subprocess = flexmock(module.subprocess) + subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() + + +BASE_PRUNE_FLAGS = ( + ('--keep-daily', '1'), + ('--keep-weekly', '2'), + ('--keep-monthly', '3'), +) + + +def test_make_prune_flags_should_return_flags_from_config(): + retention_config = OrderedDict( + ( + ('keep_daily', 1), + ('keep_weekly', 2), + ('keep_monthly', 3), + ) + ) + + result = module._make_prune_flags(retention_config) + + assert tuple(result) == BASE_PRUNE_FLAGS + + +PRUNE_COMMAND = ( + 'borg', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', +) + + +def test_prune_archives_should_call_borg_with_parameters(): + retention_config = flexmock() + flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( + BASE_PRUNE_FLAGS, + ) + insert_subprocess_mock(PRUNE_COMMAND) + + module.prune_archives( + verbosity=None, + repository='repo', + retention_config=retention_config, + ) + + +def test_prune_archives_with_verbosity_some_should_call_borg_with_info_parameter(): + retention_config = flexmock() + flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( + BASE_PRUNE_FLAGS, + ) + insert_subprocess_mock(PRUNE_COMMAND + ('--info', '--stats',)) + + module.prune_archives( + repository='repo', + verbosity=VERBOSITY_SOME, + retention_config=retention_config, + ) + + +def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_parameter(): + retention_config = flexmock() + flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( + BASE_PRUNE_FLAGS, + ) + insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats',)) + + module.prune_archives( + repository='repo', + verbosity=VERBOSITY_LOTS, + retention_config=retention_config, + ) + +def test_prune_archives_with_remote_path_should_call_borg_with_remote_path_parameters(): + retention_config = flexmock() + flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( + BASE_PRUNE_FLAGS, + ) + insert_subprocess_mock(PRUNE_COMMAND + ('--remote-path', 'borg1')) + + module.prune_archives( + verbosity=None, + repository='repo', + retention_config=retention_config, + remote_path='borg1', + ) diff --git a/borgmatic/tests/unit/test_borg.py b/borgmatic/tests/unit/test_borg.py deleted file mode 100644 index 1cba040..0000000 --- a/borgmatic/tests/unit/test_borg.py +++ /dev/null @@ -1,642 +0,0 @@ -from collections import OrderedDict -from subprocess import STDOUT -import sys -import os - -from flexmock import flexmock -import pytest - -from borgmatic import borg as module -from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS - - -def test_initialize_with_passphrase_should_set_environment(): - orig_environ = os.environ - - try: - os.environ = {} - module.initialize({'encryption_passphrase': 'pass'}, command='borg') - assert os.environ.get('BORG_PASSPHRASE') == 'pass' - finally: - os.environ = orig_environ - - -def test_initialize_without_passphrase_should_not_set_environment(): - orig_environ = os.environ - - try: - os.environ = {} - module.initialize({}, command='borg') - assert os.environ.get('BORG_PASSPHRASE') == None - finally: - os.environ = orig_environ - -def test_write_exclude_file_does_not_raise(): - temporary_file = flexmock( - name='filename', - write=lambda mode: None, - flush=lambda: None, - ) - flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file) - - module._write_exclude_file(['exclude']) - - -def test_write_exclude_file_with_empty_exclude_patterns_does_not_raise(): - module._write_exclude_file([]) - - -def insert_subprocess_mock(check_call_command, **kwargs): - subprocess = flexmock(module.subprocess) - subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() - flexmock(module).subprocess = subprocess - - -def insert_subprocess_never(): - subprocess = flexmock(module.subprocess) - subprocess.should_receive('check_call').never() - flexmock(module).subprocess = subprocess - - -def insert_subprocess_check_output_mock(check_output_command, result, **kwargs): - subprocess = flexmock(module.subprocess) - subprocess.should_receive('check_output').with_args(check_output_command, **kwargs).and_return(result).once() - flexmock(module).subprocess = subprocess - - -def insert_platform_mock(): - flexmock(module.platform).should_receive('node').and_return('host') - - -def insert_datetime_mock(): - flexmock(module).datetime = flexmock().should_receive('now').and_return( - flexmock().should_receive('isoformat').and_return('now').mock - ).mock - - -CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar') - - -def test_create_archive_should_call_borg_with_parameters(): - flexmock(module).should_receive('_write_exclude_file') - insert_subprocess_mock(CREATE_COMMAND) - insert_platform_mock() - insert_datetime_mock() - - module.create_archive( - verbosity=None, - repository='repo', - location_config={ - 'source_directories': ['foo', 'bar'], - 'repositories': ['repo'], - 'exclude_patterns': None, - }, - storage_config={}, - command='borg', - ) - - -def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): - flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes')) - insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes')) - insert_platform_mock() - insert_datetime_mock() - - module.create_archive( - verbosity=None, - repository='repo', - location_config={ - 'source_directories': ['foo', 'bar'], - 'repositories': ['repo'], - 'exclude_patterns': ['exclude'], - }, - storage_config={}, - command='borg', - ) - - -def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter(): - flexmock(module).should_receive('_write_exclude_file') - insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',)) - insert_platform_mock() - insert_datetime_mock() - - module.create_archive( - verbosity=VERBOSITY_SOME, - repository='repo', - location_config={ - 'source_directories': ['foo', 'bar'], - 'repositories': ['repo'], - 'exclude_patterns': None, - }, - storage_config={}, - command='borg', - ) - - -def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter(): - flexmock(module).should_receive('_write_exclude_file') - insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats')) - insert_platform_mock() - insert_datetime_mock() - - module.create_archive( - verbosity=VERBOSITY_LOTS, - repository='repo', - location_config={ - 'source_directories': ['foo', 'bar'], - 'repositories': ['repo'], - 'exclude_patterns': None, - }, - storage_config={}, - command='borg', - ) - - -def test_create_archive_with_compression_should_call_borg_with_compression_parameters(): - flexmock(module).should_receive('_write_exclude_file') - insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) - insert_platform_mock() - insert_datetime_mock() - - module.create_archive( - verbosity=None, - repository='repo', - location_config={ - 'source_directories': ['foo', 'bar'], - 'repositories': ['repo'], - 'exclude_patterns': None, - }, - storage_config={'compression': 'rle'}, - command='borg', - ) - - -def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters(): - flexmock(module).should_receive('_write_exclude_file') - insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) - insert_platform_mock() - insert_datetime_mock() - - module.create_archive( - verbosity=None, - repository='repo', - location_config={ - 'source_directories': ['foo', 'bar'], - 'repositories': ['repo'], - 'one_file_system': True, - 'exclude_patterns': None, - }, - storage_config={}, - command='borg', - ) - - -def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): - flexmock(module).should_receive('_write_exclude_file') - insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1')) - insert_platform_mock() - insert_datetime_mock() - - module.create_archive( - verbosity=None, - repository='repo', - location_config={ - 'source_directories': ['foo', 'bar'], - 'repositories': ['repo'], - 'remote_path': 'borg1', - 'exclude_patterns': None, - }, - storage_config={}, - command='borg', - ) - - -def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): - flexmock(module).should_receive('_write_exclude_file') - insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) - insert_platform_mock() - insert_datetime_mock() - - module.create_archive( - verbosity=None, - repository='repo', - location_config={ - 'source_directories': ['foo', 'bar'], - 'repositories': ['repo'], - 'exclude_patterns': None, - }, - storage_config={'umask': 740}, - command='borg', - ) - - -def test_create_archive_with_source_directories_glob_expands(): - flexmock(module).should_receive('_write_exclude_file') - insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) - insert_platform_mock() - insert_datetime_mock() - flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) - - module.create_archive( - verbosity=None, - repository='repo', - location_config={ - 'source_directories': ['foo*'], - 'repositories': ['repo'], - 'exclude_patterns': None, - }, - storage_config={}, - command='borg', - ) - - -def test_create_archive_with_non_matching_source_directories_glob_passes_through(): - flexmock(module).should_receive('_write_exclude_file') - insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*')) - insert_platform_mock() - insert_datetime_mock() - flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([]) - - module.create_archive( - verbosity=None, - repository='repo', - location_config={ - 'source_directories': ['foo*'], - 'repositories': ['repo'], - 'exclude_patterns': None, - }, - storage_config={}, - command='borg', - ) - - -def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): - flexmock(module).should_receive('_write_exclude_file') - insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) - insert_platform_mock() - insert_datetime_mock() - flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) - - module.create_archive( - verbosity=None, - repository='repo', - location_config={ - 'source_directories': ['foo*'], - 'repositories': ['repo'], - 'exclude_patterns': None, - }, - storage_config={}, - command='borg', - ) - - -BASE_PRUNE_FLAGS = ( - ('--keep-daily', '1'), - ('--keep-weekly', '2'), - ('--keep-monthly', '3'), -) - - -def test_make_prune_flags_should_return_flags_from_config(): - retention_config = OrderedDict( - ( - ('keep_daily', 1), - ('keep_weekly', 2), - ('keep_monthly', 3), - ) - ) - - result = module._make_prune_flags(retention_config) - - assert tuple(result) == BASE_PRUNE_FLAGS - - -PRUNE_COMMAND = ( - 'borg', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', -) - - -def test_prune_archives_should_call_borg_with_parameters(): - retention_config = flexmock() - flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( - BASE_PRUNE_FLAGS, - ) - insert_subprocess_mock(PRUNE_COMMAND) - - module.prune_archives( - verbosity=None, - repository='repo', - retention_config=retention_config, - command='borg', - ) - - -def test_prune_archives_with_verbosity_some_should_call_borg_with_info_parameter(): - retention_config = flexmock() - flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( - BASE_PRUNE_FLAGS, - ) - insert_subprocess_mock(PRUNE_COMMAND + ('--info', '--stats',)) - - module.prune_archives( - repository='repo', - verbosity=VERBOSITY_SOME, - retention_config=retention_config, - command='borg', - ) - - -def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_parameter(): - retention_config = flexmock() - flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( - BASE_PRUNE_FLAGS, - ) - insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats',)) - - module.prune_archives( - repository='repo', - verbosity=VERBOSITY_LOTS, - retention_config=retention_config, - command='borg', - ) - -def test_prune_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): - retention_config = flexmock() - flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( - BASE_PRUNE_FLAGS, - ) - insert_subprocess_mock(PRUNE_COMMAND + ('--remote-path', 'borg1')) - - module.prune_archives( - verbosity=None, - repository='repo', - retention_config=retention_config, - command='borg', - remote_path='borg1', - ) - - -def test_parse_checks_returns_them_as_tuple(): - checks = module._parse_checks({'checks': ['foo', 'disabled', 'bar']}) - - assert checks == ('foo', 'bar') - - -def test_parse_checks_with_missing_value_returns_defaults(): - checks = module._parse_checks({}) - - assert checks == module.DEFAULT_CHECKS - - -def test_parse_checks_with_blank_value_returns_defaults(): - checks = module._parse_checks({'checks': []}) - - assert checks == module.DEFAULT_CHECKS - - -def test_parse_checks_with_disabled_returns_no_checks(): - checks = module._parse_checks({'checks': ['disabled']}) - - assert checks == () - - -def test_make_check_flags_with_checks_returns_flags(): - flags = module._make_check_flags(('repository',)) - - assert flags == ('--repository-only',) - - -def test_make_check_flags_with_extract_check_does_not_make_extract_flag(): - flags = module._make_check_flags(('extract',)) - - assert flags == () - - -def test_make_check_flags_with_default_checks_returns_no_flags(): - flags = module._make_check_flags(module.DEFAULT_CHECKS) - - assert flags == () - - -def test_make_check_flags_with_checks_and_last_returns_flags_including_last(): - flags = module._make_check_flags(('repository',), check_last=3) - - assert flags == ('--repository-only', '--last', '3') - - -def test_make_check_flags_with_default_checks_and_last_returns_last_flag(): - flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3) - - assert flags == ('--last', '3') - - -@pytest.mark.parametrize( - 'checks', - ( - ('repository',), - ('archives',), - ('repository', 'archives'), - ('repository', 'archives', 'other'), - ), -) -def test_check_archives_should_call_borg_with_parameters(checks): - check_last = flexmock() - consistency_config = flexmock().should_receive('get').and_return(check_last).mock - flexmock(module).should_receive('_parse_checks').and_return(checks) - flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) - stdout = flexmock() - insert_subprocess_mock( - ('borg', 'check', 'repo'), - stdout=stdout, stderr=STDOUT, - ) - insert_platform_mock() - insert_datetime_mock() - flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout) - flexmock(module.os).should_receive('devnull') - - module.check_archives( - verbosity=None, - repository='repo', - consistency_config=consistency_config, - command='borg', - ) - - -def test_check_archives_with_extract_check_should_call_extract_only(): - checks = ('extract',) - check_last = flexmock() - consistency_config = flexmock().should_receive('get').and_return(check_last).mock - flexmock(module).should_receive('_parse_checks').and_return(checks) - flexmock(module).should_receive('_make_check_flags').never() - flexmock(module).should_receive('extract_last_archive_dry_run').once() - insert_subprocess_never() - - module.check_archives( - verbosity=None, - repository='repo', - consistency_config=consistency_config, - command='borg', - ) - - -def test_check_archives_with_verbosity_some_should_call_borg_with_info_parameter(): - checks = ('repository',) - consistency_config = flexmock().should_receive('get').and_return(None).mock - flexmock(module).should_receive('_parse_checks').and_return(checks) - flexmock(module).should_receive('_make_check_flags').and_return(()) - insert_subprocess_mock( - ('borg', 'check', 'repo', '--info'), - stdout=None, stderr=STDOUT, - ) - insert_platform_mock() - insert_datetime_mock() - - module.check_archives( - verbosity=VERBOSITY_SOME, - repository='repo', - consistency_config=consistency_config, - command='borg', - ) - - -def test_check_archives_with_verbosity_lots_should_call_borg_with_debug_parameter(): - checks = ('repository',) - consistency_config = flexmock().should_receive('get').and_return(None).mock - flexmock(module).should_receive('_parse_checks').and_return(checks) - flexmock(module).should_receive('_make_check_flags').and_return(()) - insert_subprocess_mock( - ('borg', 'check', 'repo', '--debug'), - stdout=None, stderr=STDOUT, - ) - insert_platform_mock() - insert_datetime_mock() - - module.check_archives( - verbosity=VERBOSITY_LOTS, - repository='repo', - consistency_config=consistency_config, - command='borg', - ) - - -def test_check_archives_without_any_checks_should_bail(): - consistency_config = flexmock().should_receive('get').and_return(None).mock - flexmock(module).should_receive('_parse_checks').and_return(()) - insert_subprocess_never() - - module.check_archives( - verbosity=None, - repository='repo', - consistency_config=consistency_config, - command='borg', - ) - - -def test_check_archives_with_remote_path_should_call_borg_with_remote_path_parameters(): - checks = ('repository',) - check_last = flexmock() - consistency_config = flexmock().should_receive('get').and_return(check_last).mock - flexmock(module).should_receive('_parse_checks').and_return(checks) - flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(()) - stdout = flexmock() - insert_subprocess_mock( - ('borg', 'check', 'repo', '--remote-path', 'borg1'), - stdout=stdout, stderr=STDOUT, - ) - insert_platform_mock() - insert_datetime_mock() - flexmock(sys.modules['builtins']).should_receive('open').and_return(stdout) - flexmock(module.os).should_receive('devnull') - - module.check_archives( - verbosity=None, - repository='repo', - consistency_config=consistency_config, - command='borg', - remote_path='borg1', - ) - - -def test_extract_last_archive_dry_run_should_call_borg_with_last_archive(): - flexmock(sys.stdout).encoding = 'utf-8' - insert_subprocess_check_output_mock( - ('borg', 'list', '--short', 'repo'), - result='archive1\narchive2\n'.encode('utf-8'), - ) - insert_subprocess_mock( - ('borg', 'extract', '--dry-run', 'repo::archive2'), - ) - - module.extract_last_archive_dry_run( - verbosity=None, - repository='repo', - command='borg', - ) - - -def test_extract_last_archive_dry_run_without_any_archives_should_bail(): - flexmock(sys.stdout).encoding = 'utf-8' - insert_subprocess_check_output_mock( - ('borg', 'list', '--short', 'repo'), - result='\n'.encode('utf-8'), - ) - insert_subprocess_never() - - module.extract_last_archive_dry_run( - verbosity=None, - repository='repo', - command='borg', - ) - - -def test_extract_last_archive_dry_run_with_verbosity_some_should_call_borg_with_info_parameter(): - flexmock(sys.stdout).encoding = 'utf-8' - insert_subprocess_check_output_mock( - ('borg', 'list', '--short', 'repo', '--info'), - result='archive1\narchive2\n'.encode('utf-8'), - ) - insert_subprocess_mock( - ('borg', 'extract', '--dry-run', 'repo::archive2', '--info'), - ) - - module.extract_last_archive_dry_run( - verbosity=VERBOSITY_SOME, - repository='repo', - command='borg', - ) - - -def test_extract_last_archive_dry_run_with_verbosity_lots_should_call_borg_with_debug_parameter(): - flexmock(sys.stdout).encoding = 'utf-8' - insert_subprocess_check_output_mock( - ('borg', 'list', '--short', 'repo', '--debug'), - result='archive1\narchive2\n'.encode('utf-8'), - ) - insert_subprocess_mock( - ('borg', 'extract', '--dry-run', 'repo::archive2', '--debug', '--list'), - ) - - module.extract_last_archive_dry_run( - verbosity=VERBOSITY_LOTS, - repository='repo', - command='borg', - ) - - -def test_extract_last_archive_dry_run_should_call_borg_with_remote_path_parameters(): - flexmock(sys.stdout).encoding = 'utf-8' - insert_subprocess_check_output_mock( - ('borg', 'list', '--short', 'repo', '--remote-path', 'borg1'), - result='archive1\narchive2\n'.encode('utf-8'), - ) - insert_subprocess_mock( - ('borg', 'extract', '--dry-run', 'repo::archive2', '--remote-path', 'borg1'), - ) - - module.extract_last_archive_dry_run( - verbosity=None, - repository='repo', - command='borg', - remote_path='borg1', - ) From 674a6153f38f999c2d0388021268649fb70cae39 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 5 Aug 2017 22:26:28 -0700 Subject: [PATCH 170/189] Fix imports of borg/*.py modules now that they've been split out. --- borgmatic/commands/borgmatic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index c5c7a28..5085b65 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -4,7 +4,7 @@ import os from subprocess import CalledProcessError import sys -from borgmatic import borg +from borgmatic.borg import check, create, prune from borgmatic.config import collect, convert, validate @@ -92,20 +92,20 @@ def main(): # pragma: no cover ) remote_path = location.get('remote_path') - borg.initialize(storage) + create.initialize(storage) for repository in location['repositories']: if args.prune: - borg.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) + prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) if args.create: - borg.create_archive( + create.create_archive( args.verbosity, repository, location, storage, ) if args.check: - borg.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) + check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) From 51095cd4192da82106cda54338c4242bd1bd89db Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 5 Aug 2017 22:26:38 -0700 Subject: [PATCH 171/189] Remove unused imports. --- borgmatic/commands/convert_config.py | 1 - borgmatic/commands/generate_config.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py index ca001e0..6a269c8 100644 --- a/borgmatic/commands/convert_config.py +++ b/borgmatic/commands/convert_config.py @@ -7,7 +7,6 @@ import textwrap from ruamel import yaml -from borgmatic import borg from borgmatic.config import convert, generate, legacy, validate diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py index ba88ddf..92aa797 100644 --- a/borgmatic/commands/generate_config.py +++ b/borgmatic/commands/generate_config.py @@ -4,9 +4,6 @@ import os from subprocess import CalledProcessError import sys -from ruamel import yaml - -from borgmatic import borg from borgmatic.config import convert, generate, validate From 10cac46f4c56f47b61e1b938a4f8443f39c369d6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 5 Aug 2017 23:32:39 -0700 Subject: [PATCH 172/189] #12, #35: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options. --- NEWS | 4 + README.md | 2 +- borgmatic/borg/create.py | 32 +++++++- borgmatic/config/schema.yaml | 18 +++++ borgmatic/tests/unit/borg/test_create.py | 96 +++++++++++++++++++++--- setup.py | 2 +- 6 files changed, 136 insertions(+), 18 deletions(-) diff --git a/NEWS b/NEWS index 6222fea..66447b2 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.1.6 + + * #12, #35: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options. + 1.1.5 * #34: New "extract" consistency check that performs a dry-run extraction of the most recent diff --git a/README.md b/README.md index 057388b..6840d06 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ However, see below about special cases. borgmatic changed its configuration file format in version 1.1.0 from INI-style to YAML. This better supports validation, and has a more natural way to express lists of values. To upgrade your existing configuration, first -upgrade to the new version of borgmatic: +upgrade to the new version of borgmatic. As of version 1.1.0, borgmatic no longer supports Python 2. If you were already running borgmatic with Python 3, then you can simply upgrade borgmatic diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 21af701..2696e75 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -31,12 +31,33 @@ def _write_exclude_file(exclude_patterns=None): return exclude_file +def _make_exclude_flags(location_config, exclude_patterns_filename=None): + ''' + Given a location config dict with various exclude options, and a filename containing any exclude + patterns, return the corresponding Borg flags as a tuple. + ''' + exclude_filenames = tuple(location_config.get('exclude_from', ())) + ( + (exclude_patterns_filename,) if exclude_patterns_filename else () + ) + exclude_from_flags = tuple( + itertools.chain.from_iterable( + ('--exclude-from', exclude_filename) + for exclude_filename in exclude_filenames + ) + ) + caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else () + if_present = location_config.get('exclude_if_present') + if_present_flags = ('--exclude-if-present', if_present) if if_present else () + + return exclude_from_flags + caches_flag + if_present_flags + + def create_archive( verbosity, repository, location_config, storage_config, ): ''' - Given a vebosity flag, a storage config dict, a list of source directories, a local or remote - repository path, a list of exclude patterns, create a Borg archive. + Given a vebosity flag, a local or remote repository path, a location config dict, and a storage + config dict, create a Borg archive. ''' sources = tuple( itertools.chain.from_iterable( @@ -45,8 +66,11 @@ def create_archive( ) ) - exclude_file = _write_exclude_file(location_config.get('exclude_patterns')) - exclude_flags = ('--exclude-from', exclude_file.name) if exclude_file else () + exclude_patterns_file = _write_exclude_file(location_config.get('exclude_patterns')) + exclude_flags = _make_exclude_flags( + location_config, + exclude_patterns_file.name if exclude_patterns_file else None, + ) compression = storage_config.get('compression', None) compression_flags = ('--compression', compression) if compression else () umask = storage_config.get('umask', None) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 567ec73..f7e4c4f 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -45,6 +45,24 @@ map: - '*.pyc' - /home/*/.cache - /etc/ssl + exclude_from: + seq: + - type: scalar + desc: | + Read exclude patterns from one or more separate named files, one pattern per + line. + example: + - /etc/borgmatic/excludes + exclude_caches: + type: bool + desc: | + Exclude directories that contain a CACHEDIR.TAG file. See + http://www.brynosaurus.com/cachedir/spec.html for details. + example: true + exclude_if_present: + type: scalar + desc: Exclude directories that contain a file with the given filename. + example: .nobackup storage: desc: | Repository storage options. See diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py index 3b4d268..6d2aa45 100644 --- a/borgmatic/tests/unit/borg/test_create.py +++ b/borgmatic/tests/unit/borg/test_create.py @@ -58,11 +58,72 @@ def insert_datetime_mock(): ).mock +def test_make_exclude_flags_includes_exclude_patterns_filename_when_given(): + exclude_flags = module._make_exclude_flags( + location_config={'exclude_patterns': ['*.pyc', '/var']}, + exclude_patterns_filename='/tmp/excludes', + ) + + assert exclude_flags == ('--exclude-from', '/tmp/excludes') + + +def test_make_exclude_flags_includes_exclude_from_filenames_when_in_config(): + flexmock(module).should_receive('_write_exclude_file').and_return(None) + + exclude_flags = module._make_exclude_flags( + location_config={'exclude_from': ['excludes', 'other']}, + ) + + assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', 'other') + + +def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_exclude_from_in_config(): + flexmock(module).should_receive('_write_exclude_file').and_return(None) + + exclude_flags = module._make_exclude_flags( + location_config={'exclude_from': ['excludes']}, + exclude_patterns_filename='/tmp/excludes', + ) + + assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes') + + +def test_make_exclude_flags_includes_exclude_caches_when_true_in_config(): + exclude_flags = module._make_exclude_flags( + location_config={'exclude_caches': True}, + ) + + assert exclude_flags == ('--exclude-caches',) + + +def test_make_exclude_flags_does_not_include_exclude_caches_when_false_in_config(): + exclude_flags = module._make_exclude_flags( + location_config={'exclude_caches': False}, + ) + + assert exclude_flags == () + + +def test_make_exclude_flags_includes_exclude_if_present_when_in_config(): + exclude_flags = module._make_exclude_flags( + location_config={'exclude_if_present': 'exclude_me'}, + ) + + assert exclude_flags == ('--exclude-if-present', 'exclude_me') + + +def test_make_exclude_flags_is_empty_when_config_has_no_excludes(): + exclude_flags = module._make_exclude_flags(location_config={}) + + assert exclude_flags == () + + CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar') def test_create_archive_should_call_borg_with_parameters(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND) insert_platform_mock() insert_datetime_mock() @@ -80,8 +141,10 @@ def test_create_archive_should_call_borg_with_parameters(): def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): - flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='excludes')) - insert_subprocess_mock(CREATE_COMMAND + ('--exclude-from', 'excludes')) + exclude_flags = ('--exclude-from', 'excludes') + flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes')) + flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags) + insert_subprocess_mock(CREATE_COMMAND + exclude_flags) insert_platform_mock() insert_datetime_mock() @@ -98,7 +161,8 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',)) insert_platform_mock() insert_datetime_mock() @@ -116,7 +180,8 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats')) insert_platform_mock() insert_datetime_mock() @@ -134,7 +199,8 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete def test_create_archive_with_compression_should_call_borg_with_compression_parameters(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) insert_platform_mock() insert_datetime_mock() @@ -152,7 +218,8 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) insert_platform_mock() insert_datetime_mock() @@ -171,7 +238,8 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1')) insert_platform_mock() insert_datetime_mock() @@ -190,7 +258,8 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) insert_platform_mock() insert_datetime_mock() @@ -208,7 +277,8 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): def test_create_archive_with_source_directories_glob_expands(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() @@ -227,7 +297,8 @@ def test_create_archive_with_source_directories_glob_expands(): def test_create_archive_with_non_matching_source_directories_glob_passes_through(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*')) insert_platform_mock() insert_datetime_mock() @@ -246,7 +317,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): - flexmock(module).should_receive('_write_exclude_file') + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) insert_platform_mock() insert_datetime_mock() diff --git a/setup.py b/setup.py index 2ff9690..a842ab5 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.5' +VERSION = '1.1.6' setup( From 3664ac741899c55d3dfb20a86f9da8b1046785db Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 5 Aug 2017 23:33:08 -0700 Subject: [PATCH 173/189] Added tag 1.1.6 for changeset 4daa944c122c --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 50ee67d..5859524 100644 --- a/.hgtags +++ b/.hgtags @@ -36,3 +36,4 @@ f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2 3f838f661546e04529b453aa443529b432afc243 1.1.3 3d605962d891731a0f372b903b556ac7a8c8359f 1.1.4 64ca13bfe050f656b44ed2eb1c3db045bfddd133 1.1.5 +4daa944c122c572b9b56bfcac3f4e2869181c630 1.1.6 From 37ae34a432483d1af1a33032113c568954b588c3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 26 Aug 2017 16:07:30 -0700 Subject: [PATCH 174/189] When pruning, make highest verbosity level list archives kept and pruned. --- NEWS | 4 ++++ borgmatic/borg/prune.py | 2 +- borgmatic/tests/unit/borg/test_prune.py | 2 +- setup.py | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 66447b2..677aec6 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.1.7.dev0 + + * When pruning, make highest verbosity level list archives kept and pruned. + 1.1.6 * #12, #35: Support for Borg --exclude-from, --exclude-caches, and --exclude-if-present options. diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 8f52cb4..7f0c038 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -33,7 +33,7 @@ def prune_archives(verbosity, repository, retention_config, remote_path=None): remote_path_flags = ('--remote-path', remote_path) if remote_path else () verbosity_flags = { VERBOSITY_SOME: ('--info', '--stats',), - VERBOSITY_LOTS: ('--debug', '--stats'), + VERBOSITY_LOTS: ('--debug', '--stats', '--list'), }.get(verbosity, ()) full_command = ( diff --git a/borgmatic/tests/unit/borg/test_prune.py b/borgmatic/tests/unit/borg/test_prune.py index 379960a..ffcf32a 100644 --- a/borgmatic/tests/unit/borg/test_prune.py +++ b/borgmatic/tests/unit/borg/test_prune.py @@ -70,7 +70,7 @@ def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_paramete flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( BASE_PRUNE_FLAGS, ) - insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats',)) + insert_subprocess_mock(PRUNE_COMMAND + ('--debug', '--stats', '--list')) module.prune_archives( repository='repo', diff --git a/setup.py b/setup.py index a842ab5..f70e334 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.6' +VERSION = '1.1.7.dev0' setup( From 50c4f6f2a104ed72e26e3472a087623d9812596e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 26 Aug 2017 16:13:41 -0700 Subject: [PATCH 175/189] Adding documentation note about pruning happening before archiving. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 6840d06..588c07c 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,11 @@ If you'd like to see the available command-line arguments, view the help: borgmatic --help +Note that borgmatic prunes archives *before* creating an archive, so as to +free up space for archiving. This means that when a borgmatic run finishes, +there may still be prune-able archives. Not to worry, as they will get cleaned +up at the start of the next run. + ### Verbosity By default, the backup will proceed silently except in the case of errors. But From d127e73590e85f3c93b678b36e22736d08f01010 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 26 Aug 2017 16:18:53 -0700 Subject: [PATCH 176/189] Clarification of Python 3 pip usage in documentation. --- NEWS | 1 + README.md | 19 ++++++++----------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/NEWS b/NEWS index 677aec6..2df29ec 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ 1.1.7.dev0 * When pruning, make highest verbosity level list archives kept and pruned. + * Clarification of Python 3 pip usage in documentation. 1.1.6 diff --git a/README.md b/README.md index 588c07c..1e10d44 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,10 @@ key-based ssh access to the desired user account on the remote host. To install borgmatic, run the following command to download and install it: - sudo pip install --upgrade borgmatic + sudo pip3 install --upgrade borgmatic -Make sure you're using Python 3, as borgmatic does not support Python 2. (You -may have to use "pip3" or similar instead of "pip".) +Note that your pip binary may have a different name than "pip3". Make sure +you're using Python 3, as borgmatic does not support Python 2. ## Configuration @@ -108,10 +108,7 @@ default, the traditional /etc/borgmatic/config.yaml as well. In general, all you should need to do to upgrade borgmatic is run the following: - sudo pip install --upgrade borgmatic - -(You may have to use "pip3" or similar instead of "pip", so Python 3 gets -used.) + sudo pip3 install --upgrade borgmatic However, see below about special cases. @@ -127,7 +124,7 @@ As of version 1.1.0, borgmatic no longer supports Python 2. If you were already running borgmatic with Python 3, then you can simply upgrade borgmatic in-place: - sudo pip install --upgrade borgmatic + sudo pip3 install --upgrade borgmatic But if you were running borgmatic with Python 2, uninstall and reinstall instead: @@ -162,8 +159,8 @@ your borgmatic configuration files. If you were already using Borg with atticmatic, then you can easily upgrade from atticmatic to borgmatic. Simply run the following commands: - sudo pip uninstall atticmatic - sudo pip install borgmatic + sudo pip3 uninstall atticmatic + sudo pip3 install borgmatic That's it! borgmatic will continue using your /etc/borgmatic configuration files. @@ -252,7 +249,7 @@ borgmatic to run. First install tox, which is used for setting up testing environments: - pip install tox + pip3 install tox Then, to actually run tests, run: From 3af92f8b9286d98f9c83c166772573216e13765c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 27 Aug 2017 10:01:49 -0700 Subject: [PATCH 177/189] Fix for traceback when "exclude_from" value is empty in configuration file. --- NEWS | 1 + borgmatic/borg/create.py | 2 +- borgmatic/tests/unit/borg/test_create.py | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 2df29ec..2e5abe7 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 1.1.7.dev0 + * Fix for traceback when "exclude_from" value is empty in configuration file. * When pruning, make highest verbosity level list archives kept and pruned. * Clarification of Python 3 pip usage in documentation. diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 2696e75..5073cc5 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -36,7 +36,7 @@ def _make_exclude_flags(location_config, exclude_patterns_filename=None): Given a location config dict with various exclude options, and a filename containing any exclude patterns, return the corresponding Borg flags as a tuple. ''' - exclude_filenames = tuple(location_config.get('exclude_from', ())) + ( + exclude_filenames = tuple(location_config.get('exclude_from') or ()) + ( (exclude_patterns_filename,) if exclude_patterns_filename else () ) exclude_from_flags = tuple( diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py index 6d2aa45..6f9b5a3 100644 --- a/borgmatic/tests/unit/borg/test_create.py +++ b/borgmatic/tests/unit/borg/test_create.py @@ -88,6 +88,16 @@ def test_make_exclude_flags_includes_both_filenames_when_patterns_given_and_excl assert exclude_flags == ('--exclude-from', 'excludes', '--exclude-from', '/tmp/excludes') +def test_make_exclude_flags_considers_none_exclude_from_filenames_as_empty(): + flexmock(module).should_receive('_write_exclude_file').and_return(None) + + exclude_flags = module._make_exclude_flags( + location_config={'exclude_from': None}, + ) + + assert exclude_flags == () + + def test_make_exclude_flags_includes_exclude_caches_when_true_in_config(): exclude_flags = module._make_exclude_flags( location_config={'exclude_caches': True}, From 95533d2b3103f607e541c8b5ef4d80569fd3225b Mon Sep 17 00:00:00 2001 From: Michele Lazzeri <19763535+mlazze@users.noreply.github.com> Date: Sun, 3 Sep 2017 20:13:14 +0200 Subject: [PATCH 178/189] Added storage.archive_name_format to config (#16) * Added storage.archive_name_format to config --- borgmatic/borg/create.py | 11 ++-- borgmatic/config/schema.yaml | 12 +++- borgmatic/tests/unit/borg/test_create.py | 79 +++++++++++++----------- borgmatic/tests/unit/borg/test_prune.py | 19 ++++++ 4 files changed, 78 insertions(+), 43 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 5073cc5..c2f2cf3 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -1,8 +1,6 @@ -from datetime import datetime import glob import itertools import os -import platform import subprocess import tempfile @@ -82,15 +80,16 @@ def create_archive( VERBOSITY_SOME: ('--info', '--stats',), VERBOSITY_LOTS: ('--debug', '--list', '--stats'), }.get(verbosity, ()) + default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' + archive_name_format = storage_config.get('archive_name_format', default_archive_name_format) full_command = ( 'borg', 'create', - '{repository}::{hostname}-{timestamp}'.format( + '{repository}::{archive_name_format}'.format( repository=repository, - hostname=platform.node(), - timestamp=datetime.now().isoformat(), + archive_name_format=archive_name_format, ), ) + sources + exclude_flags + compression_flags + one_file_system_flags + \ remote_path_flags + umask_flags + verbosity_flags - subprocess.check_call(full_command) + subprocess.check_call(full_command) \ No newline at end of file diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index f7e4c4f..74d99fe 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -88,6 +88,13 @@ map: type: scalar desc: Umask to be used for borg create. example: 0077 + archive_name_format: + type: scalar + desc: | + Name of the archive. Borg placeholders can be used. See + https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-placeholders + Default is "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}" + example: "{hostname}-documents-{now}" retention: desc: | Retention policy for how many backups to keep in each category. See @@ -119,7 +126,10 @@ map: example: 1 prefix: type: scalar - desc: When pruning, only consider archive names starting with this prefix. + desc: | + When pruning, only consider archive names starting with this prefix. + Borg placeholders can be used. See + https://borgbackup.readthedocs.io/en/stable/usage.html#borg-help-placeholders example: sourcehostname consistency: desc: | diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py index 6f9b5a3..743c094 100644 --- a/borgmatic/tests/unit/borg/test_create.py +++ b/borgmatic/tests/unit/borg/test_create.py @@ -48,16 +48,6 @@ def insert_subprocess_mock(check_call_command, **kwargs): subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once() -def insert_platform_mock(): - flexmock(module.platform).should_receive('node').and_return('host') - - -def insert_datetime_mock(): - flexmock(module).datetime = flexmock().should_receive('now').and_return( - flexmock().should_receive('isoformat').and_return('now').mock - ).mock - - def test_make_exclude_flags_includes_exclude_patterns_filename_when_given(): exclude_flags = module._make_exclude_flags( location_config={'exclude_patterns': ['*.pyc', '/var']}, @@ -128,15 +118,14 @@ def test_make_exclude_flags_is_empty_when_config_has_no_excludes(): assert exclude_flags == () -CREATE_COMMAND = ('borg', 'create', 'repo::host-now', 'foo', 'bar') +DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' +CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar') def test_create_archive_should_call_borg_with_parameters(): flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND) - insert_platform_mock() - insert_datetime_mock() module.create_archive( verbosity=None, @@ -155,8 +144,6 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes')) flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags) insert_subprocess_mock(CREATE_COMMAND + exclude_flags) - insert_platform_mock() - insert_datetime_mock() module.create_archive( verbosity=None, @@ -174,8 +161,6 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',)) - insert_platform_mock() - insert_datetime_mock() module.create_archive( verbosity=VERBOSITY_SOME, @@ -193,8 +178,6 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats')) - insert_platform_mock() - insert_datetime_mock() module.create_archive( verbosity=VERBOSITY_LOTS, @@ -212,8 +195,6 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) - insert_platform_mock() - insert_datetime_mock() module.create_archive( verbosity=None, @@ -231,8 +212,6 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) - insert_platform_mock() - insert_datetime_mock() module.create_archive( verbosity=None, @@ -251,8 +230,6 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1')) - insert_platform_mock() - insert_datetime_mock() module.create_archive( verbosity=None, @@ -271,8 +248,6 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) - insert_platform_mock() - insert_datetime_mock() module.create_archive( verbosity=None, @@ -289,9 +264,7 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): def test_create_archive_with_source_directories_glob_expands(): flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) - insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) - insert_platform_mock() - insert_datetime_mock() + insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')) flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( @@ -309,9 +282,7 @@ def test_create_archive_with_source_directories_glob_expands(): def test_create_archive_with_non_matching_source_directories_glob_passes_through(): flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) - insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo*')) - insert_platform_mock() - insert_datetime_mock() + insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*')) flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([]) module.create_archive( @@ -329,9 +300,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) - insert_subprocess_mock(('borg', 'create', 'repo::host-now', 'foo', 'food')) - insert_platform_mock() - insert_datetime_mock() + insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')) flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( @@ -344,3 +313,41 @@ def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): }, storage_config={}, ) + + +def test_create_archive_with_archive_name_format_without_placeholders(): + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) + insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar')) + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={ + 'archive_name_format': 'ARCHIVE_NAME', + }, + ) + + +def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): + flexmock(module).should_receive('_write_exclude_file').and_return(None) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) + insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar')) + + module.create_archive( + verbosity=None, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={ + 'archive_name_format': 'Documents_{hostname}-{now}', + }, + ) diff --git a/borgmatic/tests/unit/borg/test_prune.py b/borgmatic/tests/unit/borg/test_prune.py index ffcf32a..003b2cb 100644 --- a/borgmatic/tests/unit/borg/test_prune.py +++ b/borgmatic/tests/unit/borg/test_prune.py @@ -32,6 +32,24 @@ def test_make_prune_flags_should_return_flags_from_config(): assert tuple(result) == BASE_PRUNE_FLAGS +def test_make_prune_flags_accepts_prefix_with_placeholders(): + retention_config = OrderedDict( + ( + ('keep_daily', 1), + ('prefix', 'Documents_{hostname}-{now}'), + ) + ) + + result = module._make_prune_flags(retention_config) + + expected = ( + ('--keep-daily', '1'), + ('--prefix', 'Documents_{hostname}-{now}'), + ) + + assert tuple(result) == expected + + PRUNE_COMMAND = ( 'borg', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3', ) @@ -78,6 +96,7 @@ def test_prune_archives_with_verbosity_lots_should_call_borg_with_debug_paramete retention_config=retention_config, ) + def test_prune_archives_with_remote_path_should_call_borg_with_remote_path_parameters(): retention_config = flexmock() flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return( From bb18a9a3f20adbf89b4c9a92d8ff5ee5ebadd121 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 3 Sep 2017 11:33:07 -0700 Subject: [PATCH 179/189] Update NEWS and AUTHORS for release. --- AUTHORS | 2 ++ NEWS | 3 ++- setup.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 95d2df8..bd99af8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,4 +2,6 @@ Dan Helfman : Main developer Alexander Görtz: Python 3 compatibility Henning Schroeder: Copy editing +Michele Lazzeri: Custom archive names Robin `ypid` Schneider: Support additional options of Borg +Scott Squires: Custom archive names diff --git a/NEWS b/NEWS index 2e5abe7..04a3d5a 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ -1.1.7.dev0 +1.1.7 + * #28: Add "archive_name_format" to configuration for customizing archive names. * Fix for traceback when "exclude_from" value is empty in configuration file. * When pruning, make highest verbosity level list archives kept and pruned. * Clarification of Python 3 pip usage in documentation. diff --git a/setup.py b/setup.py index f70e334..bd93926 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.7.dev0' +VERSION = '1.1.7' setup( From 6c4f641c1ef42a44bba0e3b34531d9578c723cd1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 3 Sep 2017 11:33:10 -0700 Subject: [PATCH 180/189] Added tag 1.1.7 for changeset ec7949a14a20 --- .hgtags | 1 + 1 file changed, 1 insertion(+) diff --git a/.hgtags b/.hgtags index 5859524..23c9528 100644 --- a/.hgtags +++ b/.hgtags @@ -37,3 +37,4 @@ f052a77a8ad5a0fea7fa86a902e0e401252f7d80 1.1.2 3d605962d891731a0f372b903b556ac7a8c8359f 1.1.4 64ca13bfe050f656b44ed2eb1c3db045bfddd133 1.1.5 4daa944c122c572b9b56bfcac3f4e2869181c630 1.1.6 +ec7949a14a2051616f7cdcb8e05555f02e024ae8 1.1.7 From d30caa422e693ef1c7796899198ba9dd4f5a28d0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 8 Sep 2017 21:25:42 -0700 Subject: [PATCH 181/189] #39: Fix to make /etc/borgmatic/config.yaml optional rather than required when using the default config paths. --- NEWS | 4 ++++ borgmatic/commands/borgmatic.py | 6 ++---- borgmatic/config/collect.py | 5 ++++- .../tests/integration/commands/test_borgmatic.py | 4 ++-- borgmatic/tests/unit/config/test_collect.py | 15 ++++++++++++++- setup.py | 2 +- 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/NEWS b/NEWS index 04a3d5a..6bd9ece 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.1.8 + * #39: Fix to make /etc/borgmatic/config.yaml optional rather than required when using the default + config paths. + 1.1.7 * #28: Add "archive_name_format" to configuration for customizing archive names. diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 5085b65..2fdff34 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -9,8 +9,6 @@ from borgmatic.config import collect, convert, validate LEGACY_CONFIG_PATH = '/etc/borgmatic/config' -DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d'] -DEFAULT_EXCLUDES_PATH = '/etc/borgmatic/excludes' def parse_arguments(*arguments): @@ -30,8 +28,8 @@ def parse_arguments(*arguments): '-c', '--config', nargs='+', dest='config_paths', - default=DEFAULT_CONFIG_PATHS, - help='Configuration filenames or directories, defaults to: {}'.format(' '.join(DEFAULT_CONFIG_PATHS)), + default=collect.DEFAULT_CONFIG_PATHS, + help='Configuration filenames or directories, defaults to: {}'.format(' '.join(collect.DEFAULT_CONFIG_PATHS)), ) parser.add_argument( '--excludes', diff --git a/borgmatic/config/collect.py b/borgmatic/config/collect.py index dbe6f2f..4b8b88e 100644 --- a/borgmatic/config/collect.py +++ b/borgmatic/config/collect.py @@ -1,6 +1,9 @@ import os +DEFAULT_CONFIG_PATHS = ['/etc/borgmatic/config.yaml', '/etc/borgmatic.d'] + + def collect_config_filenames(config_paths): ''' Given a sequence of config paths, both filenames and directories, resolve that to just an @@ -14,7 +17,7 @@ def collect_config_filenames(config_paths): for path in config_paths: exists = os.path.exists(path) - if os.path.realpath(path) == '/etc/borgmatic.d' and not exists: + if os.path.realpath(path) in DEFAULT_CONFIG_PATHS and not exists: continue if not os.path.isdir(path) or not exists: diff --git a/borgmatic/tests/integration/commands/test_borgmatic.py b/borgmatic/tests/integration/commands/test_borgmatic.py index 2b82ecf..1d6aa5c 100644 --- a/borgmatic/tests/integration/commands/test_borgmatic.py +++ b/borgmatic/tests/integration/commands/test_borgmatic.py @@ -9,7 +9,7 @@ from borgmatic.commands import borgmatic as module def test_parse_arguments_with_no_arguments_uses_defaults(): parser = module.parse_arguments() - assert parser.config_paths == module.DEFAULT_CONFIG_PATHS + assert parser.config_paths == module.collect.DEFAULT_CONFIG_PATHS assert parser.excludes_filename == None assert parser.verbosity is None @@ -32,7 +32,7 @@ def test_parse_arguments_with_multiple_config_paths_parses_as_list(): def test_parse_arguments_with_verbosity_flag_overrides_default(): parser = module.parse_arguments('--verbosity', '1') - assert parser.config_paths == module.DEFAULT_CONFIG_PATHS + assert parser.config_paths == module.collect.DEFAULT_CONFIG_PATHS assert parser.excludes_filename == None assert parser.verbosity == 1 diff --git a/borgmatic/tests/unit/config/test_collect.py b/borgmatic/tests/unit/config/test_collect.py index 2adee63..c2572c8 100644 --- a/borgmatic/tests/unit/config/test_collect.py +++ b/borgmatic/tests/unit/config/test_collect.py @@ -32,6 +32,19 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno ) +def test_collect_config_filenames_skips_etc_borgmatic_config_dot_yaml_if_it_does_not_exist(): + config_paths = ('config.yaml', '/etc/borgmatic/config.yaml') + mock_path = flexmock(module.os.path) + mock_path.should_receive('exists').with_args('config.yaml').and_return(True) + mock_path.should_receive('exists').with_args('/etc/borgmatic/config.yaml').and_return(False) + mock_path.should_receive('isdir').with_args('config.yaml').and_return(False) + mock_path.should_receive('isdir').with_args('/etc/borgmatic/config.yaml').and_return(True) + + config_filenames = tuple(module.collect_config_filenames(config_paths)) + + assert config_filenames == ('config.yaml',) + + def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist(): config_paths = ('config.yaml', '/etc/borgmatic.d') mock_path = flexmock(module.os.path) @@ -45,7 +58,7 @@ def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist assert config_filenames == ('config.yaml',) -def test_collect_config_filenames_includes_directory_if_it_does_not_exist(): +def test_collect_config_filenames_includes_other_directory_if_it_does_not_exist(): config_paths = ('config.yaml', '/my/directory') mock_path = flexmock(module.os.path) mock_path.should_receive('exists').with_args('config.yaml').and_return(True) diff --git a/setup.py b/setup.py index bd93926..d3b8307 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.7' +VERSION = '1.1.8' setup( From f3d6d7c0a3674c30e4122f6cac590f3900170682 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 9 Sep 2017 17:23:31 -0700 Subject: [PATCH 182/189] #29: Support for using tilde in source directory path to reference home directory. --- NEWS | 3 ++ borgmatic/borg/create.py | 14 ++++++- borgmatic/config/schema.yaml | 3 +- borgmatic/tests/unit/borg/test_create.py | 52 +++++++++++++++++++----- setup.py | 2 +- 5 files changed, 59 insertions(+), 15 deletions(-) diff --git a/NEWS b/NEWS index 6bd9ece..1029679 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.1.9.dev0 + * #29: Support for using tilde in source directory path to reference home directory. + 1.1.8 * #39: Fix to make /etc/borgmatic/config.yaml optional rather than required when using the default config paths. diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index c2f2cf3..47315dd 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -14,6 +14,16 @@ def initialize(storage_config): os.environ['BORG_PASSPHRASE'] = passphrase +def _expand_directory(directory): + ''' + Given a directory path, expand any tilde (representing a user's home directory) and any globs + therein. Return a list of one or more resulting paths. + ''' + expanded_directory = os.path.expanduser(directory) + + return glob.glob(expanded_directory) or [expanded_directory] + + def _write_exclude_file(exclude_patterns=None): ''' Given a sequence of exclude patterns, write them to a named temporary file and return it. Return @@ -59,7 +69,7 @@ def create_archive( ''' sources = tuple( itertools.chain.from_iterable( - glob.glob(directory) or [directory] + _expand_directory(directory) for directory in location_config['source_directories'] ) ) @@ -92,4 +102,4 @@ def create_archive( ) + sources + exclude_flags + compression_flags + one_file_system_flags + \ remote_path_flags + umask_flags + verbosity_flags - subprocess.check_call(full_command) \ No newline at end of file + subprocess.check_call(full_command) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 74d99fe..d559ee3 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -12,7 +12,8 @@ map: required: true seq: - type: scalar - desc: List of source directories to backup (required). Globs are expanded. + desc: | + List of source directories to backup (required). Globs and tildes are expanded. example: - /home - /etc diff --git a/borgmatic/tests/unit/borg/test_create.py b/borgmatic/tests/unit/borg/test_create.py index 743c094..ebbee97 100644 --- a/borgmatic/tests/unit/borg/test_create.py +++ b/borgmatic/tests/unit/borg/test_create.py @@ -28,6 +28,24 @@ def test_initialize_without_passphrase_should_not_set_environment(): os.environ = orig_environ +def test_expand_directory_with_basic_path_passes_it_through(): + flexmock(module.os.path).should_receive('expanduser').and_return('foo') + flexmock(module.glob).should_receive('glob').and_return([]) + + paths = module._expand_directory('foo') + + assert paths == ['foo'] + + +def test_expand_directory_with_glob_expands(): + flexmock(module.os.path).should_receive('expanduser').and_return('foo*') + flexmock(module.glob).should_receive('glob').and_return(['foo', 'food']) + + paths = module._expand_directory('foo*') + + assert paths == ['foo', 'food'] + + def test_write_exclude_file_does_not_raise(): temporary_file = flexmock( name='filename', @@ -122,7 +140,8 @@ DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' CREATE_COMMAND = ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar') -def test_create_archive_should_call_borg_with_parameters(): +def test_create_archive_calls_borg_with_parameters(): + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND) @@ -139,8 +158,9 @@ def test_create_archive_should_call_borg_with_parameters(): ) -def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): +def test_create_archive_with_exclude_patterns_calls_borg_with_excludes(): exclude_flags = ('--exclude-from', 'excludes') + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_write_exclude_file').and_return(flexmock(name='/tmp/excludes')) flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags) insert_subprocess_mock(CREATE_COMMAND + exclude_flags) @@ -157,7 +177,8 @@ def test_create_archive_with_exclude_patterns_should_call_borg_with_excludes(): ) -def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter(): +def test_create_archive_with_verbosity_some_calls_borg_with_info_parameter(): + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--info', '--stats',)) @@ -174,7 +195,8 @@ def test_create_archive_with_verbosity_some_should_call_borg_with_info_parameter ) -def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_parameter(): +def test_create_archive_with_verbosity_lots_calls_borg_with_debug_parameter(): + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--debug', '--list', '--stats')) @@ -191,7 +213,8 @@ def test_create_archive_with_verbosity_lots_should_call_borg_with_debug_paramete ) -def test_create_archive_with_compression_should_call_borg_with_compression_parameters(): +def test_create_archive_with_compression_calls_borg_with_compression_parameters(): + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle')) @@ -208,7 +231,8 @@ def test_create_archive_with_compression_should_call_borg_with_compression_param ) -def test_create_archive_with_one_file_system_should_call_borg_with_one_file_system_parameters(): +def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters(): + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',)) @@ -226,7 +250,8 @@ def test_create_archive_with_one_file_system_should_call_borg_with_one_file_syst ) -def test_create_archive_with_remote_path_should_call_borg_with_remote_path_parameters(): +def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(): + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1')) @@ -244,7 +269,8 @@ def test_create_archive_with_remote_path_should_call_borg_with_remote_path_param ) -def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): +def test_create_archive_with_umask_calls_borg_with_umask_parameters(): + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740')) @@ -262,6 +288,7 @@ def test_create_archive_with_umask_should_call_borg_with_umask_parameters(): def test_create_archive_with_source_directories_glob_expands(): + flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')) @@ -280,6 +307,7 @@ def test_create_archive_with_source_directories_glob_expands(): def test_create_archive_with_non_matching_source_directories_glob_passes_through(): + flexmock(module).should_receive('_expand_directory').and_return(['foo*']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*')) @@ -297,11 +325,11 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through ) -def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): +def test_create_archive_with_glob_calls_borg_with_expanded_directories(): + flexmock(module).should_receive('_expand_directory').and_return(['foo', 'food']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')) - flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food']) module.create_archive( verbosity=None, @@ -315,7 +343,8 @@ def test_create_archive_with_glob_should_call_borg_with_expanded_directories(): ) -def test_create_archive_with_archive_name_format_without_placeholders(): +def test_create_archive_with_archive_name_format_calls_borg_with_archive_name(): + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar')) @@ -335,6 +364,7 @@ def test_create_archive_with_archive_name_format_without_placeholders(): def test_create_archive_with_archive_name_format_accepts_borg_placeholders(): + flexmock(module).should_receive('_expand_directory').and_return(['foo']).and_return(['bar']) flexmock(module).should_receive('_write_exclude_file').and_return(None) flexmock(module).should_receive('_make_exclude_flags').and_return(()) insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar')) diff --git a/setup.py b/setup.py index d3b8307..d23552b 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.1.8' +VERSION = '1.1.9.dev0' setup( From bd196c1fb9317a87ea3e9911277d435261aebd26 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 9 Sep 2017 17:38:14 -0700 Subject: [PATCH 183/189] Removing "from __future__ import print_function". This isn't Python 2 anymore, Toto. --- borgmatic/commands/borgmatic.py | 1 - borgmatic/commands/convert_config.py | 1 - borgmatic/commands/generate_config.py | 1 - 3 files changed, 3 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 2fdff34..b7c53a3 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -1,4 +1,3 @@ -from __future__ import print_function from argparse import ArgumentParser import os from subprocess import CalledProcessError diff --git a/borgmatic/commands/convert_config.py b/borgmatic/commands/convert_config.py index 6a269c8..399a025 100644 --- a/borgmatic/commands/convert_config.py +++ b/borgmatic/commands/convert_config.py @@ -1,4 +1,3 @@ -from __future__ import print_function from argparse import ArgumentParser import os from subprocess import CalledProcessError diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py index 92aa797..b0776e0 100644 --- a/borgmatic/commands/generate_config.py +++ b/borgmatic/commands/generate_config.py @@ -1,4 +1,3 @@ -from __future__ import print_function from argparse import ArgumentParser import os from subprocess import CalledProcessError From 86511deac4ca0afb6c357dbbf950c8f09fadd09e Mon Sep 17 00:00:00 2001 From: b3vis Date: Thu, 26 Oct 2017 05:24:24 +0100 Subject: [PATCH 184/189] Added section about docker (#18) --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 1e10d44..6e370f2 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,11 @@ To install borgmatic, run the following command to download and install it: Note that your pip binary may have a different name than "pip3". Make sure you're using Python 3, as borgmatic does not support Python 2. +### Docker + +If you would like to run borgmatic within docker please take a look at +[b3vis/borgmatic](https://hub.docker.com/r/b3vis/borgmatic/) for more information. + ## Configuration After you install borgmatic, generate a sample configuration file: From f1a98d82c6e40754396cec4770effe74523773f0 Mon Sep 17 00:00:00 2001 From: Johannes Feichtner Date: Thu, 26 Oct 2017 06:38:27 +0200 Subject: [PATCH 185/189] #16, #38: Support for user-defined hooks before/after backup, or on error. --- AUTHORS | 1 + borgmatic/commands/borgmatic.py | 38 ++++++++++++++++---------- borgmatic/commands/hook.py | 7 +++++ borgmatic/config/generate.py | 2 +- borgmatic/config/schema.yaml | 25 +++++++++++++++++ borgmatic/config/validate.py | 2 +- borgmatic/tests/unit/borg/test_hook.py | 10 +++++++ 7 files changed, 68 insertions(+), 17 deletions(-) create mode 100644 borgmatic/commands/hook.py create mode 100644 borgmatic/tests/unit/borg/test_hook.py diff --git a/AUTHORS b/AUTHORS index bd99af8..81ae45e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -5,3 +5,4 @@ Henning Schroeder: Copy editing Michele Lazzeri: Custom archive names Robin `ypid` Schneider: Support additional options of Borg Scott Squires: Custom archive names +Johannes Feichtner: Support for user hooks diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 2fdff34..e67fe88 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -5,6 +5,7 @@ from subprocess import CalledProcessError import sys from borgmatic.borg import check, create, prune +from borgmatic.commands import hook from borgmatic.config import collect, convert, validate @@ -84,26 +85,33 @@ def main(): # pragma: no cover for config_filename in config_filenames: config = validate.parse_configuration(config_filename, validate.schema_filename()) - (location, storage, retention, consistency) = ( + (location, storage, retention, consistency, hooks) = ( config.get(section_name, {}) - for section_name in ('location', 'storage', 'retention', 'consistency') + for section_name in ('location', 'storage', 'retention', 'consistency', 'hooks') ) remote_path = location.get('remote_path') - create.initialize(storage) + try: + create.initialize(storage) + hook.execute_hook(hooks.get('before_backup')) - for repository in location['repositories']: - if args.prune: - prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) - if args.create: - create.create_archive( - args.verbosity, - repository, - location, - storage, - ) - if args.check: - check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) + for repository in location['repositories']: + if args.prune: + prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) + if args.create: + create.create_archive( + args.verbosity, + repository, + location, + storage, + ) + if args.check: + check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) + + hook.execute_hook(hooks.get('after_backup')) + except (OSError, CalledProcessError): + hook.execute_hook(hooks.get('on_error')) + raise except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/borgmatic/commands/hook.py b/borgmatic/commands/hook.py new file mode 100644 index 0000000..4417dea --- /dev/null +++ b/borgmatic/commands/hook.py @@ -0,0 +1,7 @@ +import subprocess + + +def execute_hook(commands): + if commands: + for cmd in commands: + subprocess.check_call(cmd, shell=True) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 1a6d1a8..2bfc3a2 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -24,7 +24,7 @@ def _schema_to_sample_configuration(schema, level=0): for each section based on the schema "desc" description. ''' example = schema.get('example') - if example: + if example is not None: return example config = yaml.comments.CommentedMap([ diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index d559ee3..60f6cdf 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -157,3 +157,28 @@ map: desc: Restrict the number of checked archives to the last n. Applies only to the "archives" check. example: 3 + hooks: + desc: | + Shell commands or scripts to execute before and after a backup or if an error has occurred. + IMPORTANT: All provided commands and scripts are executed with user permissions of borgmatic. + Do not forget to set secure permissions on this file as well as on any script listed (chmod 0700) to + prevent potential shell injection or privilege escalation. + map: + before_backup: + seq: + - type: scalar + desc: List of one or more shell commands or scripts to execute before creating a backup. + example: + - echo "`date` - Starting a backup job." + after_backup: + seq: + - type: scalar + desc: List of one or more shell commands or scripts to execute after creating a backup. + example: + - echo "`date` - Backup created." + on_error: + seq: + - type: scalar + desc: List of one or more shell commands or scripts to execute in case an exception has occurred. + example: + - echo "`date` - Error while creating a backup." diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 3125176..ff61634 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -48,7 +48,7 @@ def parse_configuration(config_filename, schema_filename): # simply remove all examples before passing the schema to pykwalify. for section_name, section_schema in schema['map'].items(): for field_name, field_schema in section_schema['map'].items(): - field_schema.pop('example') + field_schema.pop('example', None) validator = pykwalify.core.Core(source_data=config, schema_data=schema) parsed_result = validator.validate(raise_exception=False) diff --git a/borgmatic/tests/unit/borg/test_hook.py b/borgmatic/tests/unit/borg/test_hook.py new file mode 100644 index 0000000..6aabc57 --- /dev/null +++ b/borgmatic/tests/unit/borg/test_hook.py @@ -0,0 +1,10 @@ +from flexmock import flexmock + +from borgmatic.commands import hook as module + + +def test_execute_hook_invokes_each_command(): + subprocess = flexmock(module.subprocess) + subprocess.should_receive('check_call').with_args(':', shell=True).once() + + module.execute_hook([':']) From 5c229639f0a3a581264400268b2aee2a776aa568 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 25 Oct 2017 21:47:33 -0700 Subject: [PATCH 186/189] Improve clarity of logging spew at high verbosity levels. --- NEWS | 2 ++ borgmatic/commands/borgmatic.py | 12 ++++++++++++ borgmatic/config/validate.py | 2 ++ borgmatic/verbosity.py | 15 +++++++++++++++ 4 files changed, 31 insertions(+) diff --git a/NEWS b/NEWS index 1029679..cfac2f2 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 1.1.9.dev0 + * #16, #38: Support for user-defined hooks before/after backup, or on error. + * #33: Improve clarity of logging spew at high verbosity levels. * #29: Support for using tilde in source directory path to reference home directory. 1.1.8 diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index b7c53a3..28d1e0d 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -1,10 +1,15 @@ from argparse import ArgumentParser +import logging import os from subprocess import CalledProcessError import sys from borgmatic.borg import check, create, prune from borgmatic.config import collect, convert, validate +from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS, verbosity_to_log_level + + +logger = logging.getLogger(__name__) LEGACY_CONFIG_PATH = '/etc/borgmatic/config' @@ -75,13 +80,17 @@ def parse_arguments(*arguments): def main(): # pragma: no cover try: args = parse_arguments(*sys.argv[1:]) + logging.basicConfig(level=verbosity_to_log_level(args.verbosity), format='%(message)s') + config_filenames = tuple(collect.collect_config_filenames(args.config_paths)) + logger.debug('Ensuring legacy configuration is upgraded') convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames) if len(config_filenames) == 0: raise ValueError('Error: No configuration files found in: {}'.format(' '.join(args.config_paths))) for config_filename in config_filenames: + logger.info('{}: Parsing configuration file'.format(config_filename)) config = validate.parse_configuration(config_filename, validate.schema_filename()) (location, storage, retention, consistency) = ( config.get(section_name, {}) @@ -93,8 +102,10 @@ def main(): # pragma: no cover for repository in location['repositories']: if args.prune: + logger.info('{}: Pruning archives'.format(repository)) prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) if args.create: + logger.info('{}: Creating archive'.format(repository)) create.create_archive( args.verbosity, repository, @@ -102,6 +113,7 @@ def main(): # pragma: no cover storage, ) if args.check: + logger.info('{}: Running consistency checks'.format(repository)) check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 3125176..0bac596 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -38,6 +38,8 @@ def parse_configuration(config_filename, schema_filename): Raise FileNotFoundError if the file does not exist, PermissionError if the user does not have permissions to read the file, or Validation_error if the config does not match the schema. ''' + logging.getLogger('pykwalify').setLevel(logging.ERROR) + try: config = yaml.round_trip_load(open(config_filename)) schema = yaml.round_trip_load(open(schema_filename)) diff --git a/borgmatic/verbosity.py b/borgmatic/verbosity.py index 06dfc4c..13b1ecc 100644 --- a/borgmatic/verbosity.py +++ b/borgmatic/verbosity.py @@ -1,2 +1,17 @@ +import logging + + VERBOSITY_SOME = 1 VERBOSITY_LOTS = 2 + + +def verbosity_to_log_level(verbosity): + ''' + Given a borgmatic verbosity value, return the corresponding Python log level. + ''' + return { + VERBOSITY_SOME: logging.INFO, + VERBOSITY_LOTS: logging.DEBUG, + }.get(verbosity, logging.ERROR) + + From cc78223164883465170e1066ba02010420fc9ddf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 25 Oct 2017 21:58:02 -0700 Subject: [PATCH 187/189] Fixing inconsistent indentation. --- borgmatic/commands/borgmatic.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 14d73bb..c352b9d 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -104,21 +104,21 @@ def main(): # pragma: no cover create.initialize(storage) hook.execute_hook(hooks.get('before_backup')) - for repository in location['repositories']: - if args.prune: - logger.info('{}: Pruning archives'.format(repository)) - prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) - if args.create: - logger.info('{}: Creating archive'.format(repository)) - create.create_archive( - args.verbosity, - repository, - location, - storage, - ) - if args.check: - logger.info('{}: Running consistency checks'.format(repository)) - check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) + for repository in location['repositories']: + if args.prune: + logger.info('{}: Pruning archives'.format(repository)) + prune.prune_archives(args.verbosity, repository, retention, remote_path=remote_path) + if args.create: + logger.info('{}: Creating archive'.format(repository)) + create.create_archive( + args.verbosity, + repository, + location, + storage, + ) + if args.check: + logger.info('{}: Running consistency checks'.format(repository)) + check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) hook.execute_hook(hooks.get('after_backup')) except (OSError, CalledProcessError): From a09c9f248e7c25200a9d7102ead87a619f5fde83 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 25 Oct 2017 22:32:06 -0700 Subject: [PATCH 188/189] Adding logging to hook execution! --- borgmatic/commands/borgmatic.py | 6 +-- borgmatic/commands/hook.py | 21 ++++++++-- .../tests/integration/config/test_validate.py | 42 +++++++++++++++++-- borgmatic/tests/unit/borg/test_hook.py | 14 ++++++- 4 files changed, 71 insertions(+), 12 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index c352b9d..8841b70 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -102,7 +102,7 @@ def main(): # pragma: no cover try: create.initialize(storage) - hook.execute_hook(hooks.get('before_backup')) + hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup') for repository in location['repositories']: if args.prune: @@ -120,9 +120,9 @@ def main(): # pragma: no cover logger.info('{}: Running consistency checks'.format(repository)) check.check_archives(args.verbosity, repository, consistency, remote_path=remote_path) - hook.execute_hook(hooks.get('after_backup')) + hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup') except (OSError, CalledProcessError): - hook.execute_hook(hooks.get('on_error')) + hook.execute_hook(hooks.get('on_error'), config_filename, 'on-error') raise except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) diff --git a/borgmatic/commands/hook.py b/borgmatic/commands/hook.py index 4417dea..c6f5a29 100644 --- a/borgmatic/commands/hook.py +++ b/borgmatic/commands/hook.py @@ -1,7 +1,20 @@ +import logging import subprocess -def execute_hook(commands): - if commands: - for cmd in commands: - subprocess.check_call(cmd, shell=True) +logger = logging.getLogger(__name__) + + +def execute_hook(commands, config_filename, description): + if not commands: + logger.debug('{}: No commands to run for {} hook'.format(config_filename, description)) + return + + if len(commands) == 1: + logger.info('{}: Running command for {} hook'.format(config_filename, description)) + else: + logger.info('{}: Running {} commands for {} hook'.format(config_filename, len(commands), description)) + + for command in commands: + logger.debug('{}: Hook command: {}'.format(config_filename, command)) + subprocess.check_call(command, shell=True) diff --git a/borgmatic/tests/integration/config/test_validate.py b/borgmatic/tests/integration/config/test_validate.py index 9b63ccc..eec282c 100644 --- a/borgmatic/tests/integration/config/test_validate.py +++ b/borgmatic/tests/integration/config/test_validate.py @@ -15,13 +15,18 @@ def test_schema_filename_returns_plausable_path(): assert schema_path.endswith('/schema.yaml') -def mock_config_and_schema(config_yaml): +def mock_config_and_schema(config_yaml, schema_yaml=None): ''' - Set up mocks for the config config YAML string and the default schema so that the code under - test consumes them when parsing the configuration. + Set up mocks for the given config config YAML string and the schema YAML string, or the default + schema if no schema is provided. The idea is that that the code under test consumes these mocks + when parsing the configuration. ''' config_stream = io.StringIO(config_yaml) - schema_stream = open(module.schema_filename()) + if schema_yaml is None: + schema_stream = open(module.schema_filename()) + else: + schema_stream = io.StringIO(schema_yaml) + builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args('config.yaml').and_return(config_stream) builtins.should_receive('open').with_args('schema.yaml').and_return(schema_stream) @@ -81,6 +86,35 @@ def test_parse_configuration_passes_through_quoted_punctuation(): } +def test_parse_configuration_with_schema_lacking_examples_does_not_raise(): + mock_config_and_schema( + ''' + location: + source_directories: + - /home + + repositories: + - hostname.borg + ''', + ''' + map: + location: + required: true + map: + source_directories: + required: true + seq: + - type: scalar + repositories: + required: true + seq: + - type: scalar + ''' + ) + + module.parse_configuration('config.yaml', 'schema.yaml') + + def test_parse_configuration_raises_for_missing_config_file(): with pytest.raises(FileNotFoundError): module.parse_configuration('config.yaml', 'schema.yaml') diff --git a/borgmatic/tests/unit/borg/test_hook.py b/borgmatic/tests/unit/borg/test_hook.py index 6aabc57..f87edda 100644 --- a/borgmatic/tests/unit/borg/test_hook.py +++ b/borgmatic/tests/unit/borg/test_hook.py @@ -7,4 +7,16 @@ def test_execute_hook_invokes_each_command(): subprocess = flexmock(module.subprocess) subprocess.should_receive('check_call').with_args(':', shell=True).once() - module.execute_hook([':']) + module.execute_hook([':'], 'config.yaml', 'pre-backup') + + +def test_execute_hook_with_multiple_commands_invokes_each_command(): + subprocess = flexmock(module.subprocess) + subprocess.should_receive('check_call').with_args(':', shell=True).once() + subprocess.should_receive('check_call').with_args('true', shell=True).once() + + module.execute_hook([':', 'true'], 'config.yaml', 'pre-backup') + + +def test_execute_hook_with_empty_commands_does_not_raise(): + module.execute_hook([], 'config.yaml', 'post-backup') From 2ae8ac2947bfd651a43af042c5042b414c4c50c7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 25 Oct 2017 22:36:23 -0700 Subject: [PATCH 189/189] Add tests for verbosity mapping. --- borgmatic/tests/unit/test_verbosity.py | 11 +++++++++++ borgmatic/verbosity.py | 2 -- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 borgmatic/tests/unit/test_verbosity.py diff --git a/borgmatic/tests/unit/test_verbosity.py b/borgmatic/tests/unit/test_verbosity.py new file mode 100644 index 0000000..685aed0 --- /dev/null +++ b/borgmatic/tests/unit/test_verbosity.py @@ -0,0 +1,11 @@ +import logging + +from borgmatic import verbosity as module + + +def test_verbosity_to_log_level_maps_known_verbosity_to_log_level(): + assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO + + +def test_verbosity_to_log_level_maps_unknown_verbosity_to_error_level(): + assert module.verbosity_to_log_level('my pants') == logging.ERROR diff --git a/borgmatic/verbosity.py b/borgmatic/verbosity.py index 13b1ecc..e1ea796 100644 --- a/borgmatic/verbosity.py +++ b/borgmatic/verbosity.py @@ -13,5 +13,3 @@ def verbosity_to_log_level(verbosity): VERBOSITY_SOME: logging.INFO, VERBOSITY_LOTS: logging.DEBUG, }.get(verbosity, logging.ERROR) - -