Hello community, here is the log from the commit of package duplicity for openSUSE:Factory checked in at 2017-09-07 22:10:24 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/duplicity (Old) and /work/SRC/openSUSE:Factory/.duplicity.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "duplicity" Thu Sep 7 22:10:24 2017 rev:37 rq:520502 version:0.7.14 Changes: -------- --- /work/SRC/openSUSE:Factory/duplicity/duplicity.changes 2017-07-08 12:26:20.486650438 +0200 +++ /work/SRC/openSUSE:Factory/.duplicity.new/duplicity.changes 2017-09-07 22:10:27.502279431 +0200 @@ -1,0 +2,24 @@ +Sun Sep 3 13:50:07 UTC 2017 - [email protected] + +- update to 0.7.14 + * collection-status should not sync metadata + syncing metadata might require to download several GBs + * Fixed slowness in 'collection-status' by basing the status on the + remote system only. The local cache is treated as empty. + * Fixed encrypted remote manifest handling to merely put out a non-fatal + error message and continue if the private key is not available. + * giobackend: handle a wider variety of gio backends by making + less assumptions; in particular, this fixes the google-drive: backend + * Fixed PEP8 errors in bin/duplicity + * gio: be slightly more correct and get child GFiles based on display name + * log.Warn was invoked with log.warn in webdavbackend.py + * Support gpg versions numbers that have tags on them. + * uses megatools from https://megatools.megous.com/ instead of + mega.py library which has been deprecated + * Fixed bug #1713640 with patch from Aleksandar Ivanisevic + replace 2.7 syntax with 2.6 equivalent + * Fixed bug #1638033 Remove leading slash on --file-to-restore + - code already used rstrip('/') so change to just strip('/') + * find all details on http://duplicity.nongnu.org/CHANGELOG + +------------------------------------------------------------------- Old: ---- duplicity-0.7.13.1.tar.gz New: ---- duplicity-0.7.14.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ duplicity.spec ++++++ --- /var/tmp/diff_new_pack.O5XnDR/_old 2017-09-07 22:10:28.458144706 +0200 +++ /var/tmp/diff_new_pack.O5XnDR/_new 2017-09-07 22:10:28.458144706 +0200 @@ -19,7 +19,7 @@ %{!?python_sitelib: %global python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} %{!?python_sitearch: %global python_sitearch %(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} Name: duplicity -Version: 0.7.13.1 +Version: 0.7.14 Release: 0 Summary: Encrypted bandwidth-efficient backup using the rsync algorithm License: GPL-3.0+ ++++++ duplicity-0.7.13.1.tar.gz -> duplicity-0.7.14.tar.gz ++++++ ++++ 1749 lines of diff (skipped) ++++ retrying with extended exclude list diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/CHANGELOG new/duplicity-0.7.14/CHANGELOG --- old/duplicity-0.7.13.1/CHANGELOG 2017-06-18 17:29:18.000000000 +0200 +++ new/duplicity-0.7.14/CHANGELOG 2017-08-31 14:06:41.000000000 +0200 @@ -1,3 +1,47 @@ +New in v0.7.14 (2017/08/31) +--------------------------- +* Merged in lp:~dawgfoto/duplicity/skip_sync_collection_status + - collection-status should not sync metadata + - up-to-date local metadata is not needed as collection-status is + generated from remote file list + - syncing metadata might require to download several GBs +* Fixed slowness in 'collection-status' by basing the status on the + remote system only. The local cache is treated as empty. +* Fixed encrypted remote manifest handling to merely put out a non-fatal + error message and continue if the private key is not available. +* Patched in lp:~mterry/duplicity/giobackend-display-name + - giobackend: handle a wider variety of gio backends by making less assumptions; + in particular, this fixes the google-drive: backend +* Fixed bug #1709047 with suggestion from Gary Hasson + - fixed so default was to use original filename +* Fixed PEP8 errors in bin/duplicity +* Merged in lp:~mterry/duplicity/gio_child_for_display_name_0.7 + - gio: be slightly more correct and get child GFiles based on display name +* Fixed bug #1711905 with suggestion from Schneider + - log.Warn was invoked with log.warn in webdavbackend.py +* Merged in lp:~mterry/duplicity/gpg-tag-versions + - Support gpg versions numbers that have tags on them. + - This can happen if you build gpg from git trunk (e.g. 2.1.15-beta20). Or if you run + against the freedesktop flatpak runtime (e.g. 2.1.14-unknown). +* Fixed bug #1394386 with new module megabackend.py from Tomas Vondra + - uses megatools from https://megatools.megous.com/ instead of mega.py library + which has been deprecated + - fixed copyright and PEP8 issues + - replaced subprocess.call() with self.subprocess_popen() to standardize +* Fixed bug #1713640 with patch from Aleksandar Ivanisevic + - replace 2.7 syntax with 2.6 equivalent +* Fixed bug #1538333 Assertion error in manifest.py: assert filecount == ... + - Made sure to never pass .part files as true manifest files + - Changed assert to log.Error to warn about truncated/corrupt filelist + - Added unit test to make sure detection works + - Note: while this condition is serious, it will not affect the basic backup and restore + functions. Interactive options like --list-files-changed and --file-changed will not + work correctly for this backup set, so it is advised to run a full backup as soon as + possible after this error occurs. +* Fixed bug #1638033 Remove leading slash on --file-to-restore + - code already used rstrip('/') so change to just strip('/') + + New in v0.7.13.1 (2017/06/18) ----------------------------- * Fixed problem in dist/makedist when building on Mac where AppleDouble diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/Changelog.GNU new/duplicity-0.7.14/Changelog.GNU --- old/duplicity-0.7.13.1/Changelog.GNU 2017-06-18 17:31:48.000000000 +0200 +++ new/duplicity-0.7.14/Changelog.GNU 2017-08-31 14:06:14.000000000 +0200 @@ -1,3 +1,68 @@ +2017-08-31 Kenneth Loafman <[email protected]> + + * Fixed bug #1538333 Assertion error in manifest.py: assert filecount == ... + - Made sure to never pass .part files as true manifest files + - Changed assert to log.Error to warn about truncated/corrupt filelist + - Added unit test to make sure detection works + - Note: while this condition is serious, it will not affect the basic backup and restore + functions. Interactive options like --list-files-changed and --file-changed will not + work correctly for this backup set, so it is advised to run a full backup as soon as + possible after this error occurs. + * Fixed bug #1638033 Remove leading slash on --file-to-restore + - code already used rstrip('/') so change to just strip('/') + * Prep for 0.7.14 + +2017-08-29 Kenneth Loafman <[email protected]> + + * Fixed bug #1394386 with new module megabackend.py from Tomas Vondra + - uses megatools from https://megatools.megous.com/ instead of mega.py library + which has been deprecated + - fixed copyright and PEP8 issues + - replaced subprocess.call() with self.subprocess_popen() to standardize + * Fixed bug #1713640 with patch from Aleksandar Ivanisevic + - replace 2.7 syntax with 2.6 equivalent + +2017-08-28 Kenneth Loafman <[email protected]> + + * Fixed bug #1711905 with suggestion from Schneider + - log.Warn was invoked with log.warn in webdavbackend.py + * Merged in lp:~mterry/duplicity/gpg-tag-versions + - Support gpg versions numbers that have tags on them. + - This can happen if you build gpg from git trunk (e.g. 2.1.15-beta20). Or if you run + against the freedesktop flatpak runtime (e.g. 2.1.14-unknown). + +2017-08-15 Kenneth Loafman <[email protected]> + + * Fixed bug #1709047 with suggestion from Gary Hasson + - fixed so default was to use original filename + * Fixed PEP8 errors in bin/duplicity + * Merged in lp:~mterry/duplicity/gio_child_for_display_name_0.7 + - gio: be slightly more correct and get child GFiles based on display name + +2017-08-06 Kenneth Loafman <[email protected]> + + * Patched in lp:~mterry/duplicity/giobackend-display-name + - giobackend: handle a wider variety of gio backends by making less assumptions; + in particular, this fixes the google-drive: backend + +2017-07-20 Kenneth Loafman <[email protected]> + + * Fixed encrypted remote manifest handling to merely put out a non-fatal + error message and continue if the private key is not available. + +2017-07-18 Kenneth Loafman <[email protected]> + + * Fixed slowness in 'collection-status' by basing the status on the + remote system only. The local cache is treated as empty. + +2017-07-11 Kenneth Loafman <[email protected]> + + * Merged in lp:~dawgfoto/duplicity/skip_sync_collection_status + - collection-status should not sync metadata + - up-to-date local metadata is not needed as collection-status is + generated from remote file list + - syncing metadata might require to download several GBs + 2017-06-18 Kenneth Loafman <[email protected]> * Fixed problem in dist/makedist when building on Mac where AppleDouble diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/bin/duplicity new/duplicity-0.7.14/bin/duplicity --- old/duplicity-0.7.13.1/bin/duplicity 2017-06-18 17:36:30.000000000 +0200 +++ new/duplicity-0.7.14/bin/duplicity 2017-08-31 14:25:19.000000000 +0200 @@ -2,7 +2,7 @@ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # duplicity -- Encrypted bandwidth efficient backup -# Version 0.7.13.1 released June 18, 2017 +# Version 0.7.14 released August 31, 2017 # # Copyright 2002 Ben Escoto <[email protected]> # Copyright 2007 Kenneth Loafman <[email protected]> @@ -63,13 +63,14 @@ from duplicity import util from duplicity import progress + if '--pydevd' in sys.argv or os.getenv('PYDEVD', None): # The following is for starting remote debugging in Eclipse with Pydev. # Adjust the path to your location and version of Eclipse and Pydev. if platform.platform().startswith('Linux'): - pysrc = "/opt/liclipse/plugins/org.python.pydev_5.4.0.201611281405/pysrc" + pysrc = "/opt/liclipse/plugins/org.python.pydev_5.7.0.201704111135/pysrc" elif platform.platform().startswith('Darwin'): - pysrc = "/Applications/LiClipse.app/Contents/liclipse/plugins/org.python.pydev_5.4.0.201611281405/pysrc" + pysrc = "/Applications/LiClipse.app/Contents/liclipse/plugins/org.python.pydev_5.7.0.201704111135/pysrc" else: raise Exception("Platform %s not supported by pydevd." % platform.platform()) sys.path.append(pysrc) @@ -1004,7 +1005,7 @@ _("Rerun command with --force option to actually delete.")) -def sync_archive(decrypt): +def sync_archive(): """ Synchronize local archive manifest file and sig chains to remote archives. Copy missing files from remote to local as needed to make sure the local @@ -1183,11 +1184,8 @@ if not globals.dry_run: log.Notice(_("Synchronizing remote metadata to local cache...")) if local_missing and (rem_needpass or loc_needpass): - if decrypt: - # password for the --encrypt-key - globals.gpg_profile.passphrase = get_passphrase(1, "sync") - else: - local_missing = [] # don't download if we can't decrypt + # password for the --encrypt-key + globals.gpg_profile.passphrase = get_passphrase(1, "sync") for fn in local_spurious: remove_local(fn) if hasattr(globals.backend, 'pre_process_download'): @@ -1275,7 +1273,7 @@ log Python, duplicity, and system versions """ log.Log(u'=' * 80, verbosity) - log.Log(u"duplicity 0.7.13.1 (June 18, 2017)", verbosity) + log.Log(u"duplicity 0.7.14 (August 31, 2017)", verbosity) log.Log(u"Args: %s" % util.ufn(' '.join(sys.argv)), verbosity) log.Log(u' '.join(platform.uname()), verbosity) log.Log(u"%s %s" % (sys.executable or sys.platform, sys.version), verbosity) @@ -1402,12 +1400,13 @@ check_resources(action) # check archive synch with remote, fix if needed - decrypt = action not in ["collection-status"] - sync_archive(decrypt) + if action not in ["collection-status"]: + sync_archive() # get current collection status col_stats = collections.CollectionsStatus(globals.backend, - globals.archive_dir).set_values() + globals.archive_dir, + action).set_values() while True: # if we have to clean up the last partial, then col_stats are invalidated @@ -1437,7 +1436,8 @@ log.Notice(_("Cleaning up previous partial %s backup set, restarting." % action)) last_backup.delete() col_stats = collections.CollectionsStatus(globals.backend, - globals.archive_dir).set_values() + globals.archive_dir, + action).set_values() continue break break @@ -1476,7 +1476,7 @@ elif action == "remove-all-but-n-full" or action == "remove-all-inc-of-but-n-full": remove_all_but_n_full(col_stats) elif action == "sync": - sync_archive(True) + sync_archive() else: assert action == "inc" or action == "full", action # the passphrase for full and inc is used by --sign-key @@ -1535,10 +1535,24 @@ finally: tempdir.default().cleanup() + if __name__ == "__main__": try: + + # import cProfile + # import pstats + # import StringIO + # prof = cProfile.Profile() + # prof.enable(subcalls=True, builtins=True) + with_tempdir(main) + # prof.disable() + # s = StringIO.StringIO() + # ps = pstats.Stats(prof, stream=s).sort_stats('cumulative') + # ps.print_stats(20) + # print s.getvalue() + # Don't move this lower. In order to get an exit # status out of the system, you have to call the # sys.exit() function. Python handles this by @@ -1547,7 +1561,7 @@ except SystemExit as e: # No traceback, just get out util.release_lockfile() - sys.exit(e) + sys.exit(e.code) except KeyboardInterrupt as e: # No traceback, just get out diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/bin/duplicity.1 new/duplicity-0.7.14/bin/duplicity.1 --- old/duplicity-0.7.13.1/bin/duplicity.1 2017-06-18 17:36:30.000000000 +0200 +++ new/duplicity-0.7.14/bin/duplicity.1 2017-08-31 14:25:19.000000000 +0200 @@ -1,4 +1,4 @@ -.TH DUPLICITY 1 "June 18, 2017" "Version 0.7.13.1" "User Manuals" \" -*- nroff -*- +.TH DUPLICITY 1 "August 31, 2017" "Version 0.7.14" "User Manuals" \" -*- nroff -*- .\" disable justification (adjust text to left margin only) .\" command line examples stay readable through that .ad l diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/bin/rdiffdir new/duplicity-0.7.14/bin/rdiffdir --- old/duplicity-0.7.13.1/bin/rdiffdir 2017-06-18 17:36:30.000000000 +0200 +++ new/duplicity-0.7.14/bin/rdiffdir 2017-08-31 14:25:19.000000000 +0200 @@ -1,6 +1,6 @@ #!/usr/bin/env python2 # rdiffdir -- Extend rdiff functionality to directories -# Version 0.7.13.1 released June 18, 2017 +# Version 0.7.14 released August 31, 2017 # # Copyright 2002 Ben Escoto <[email protected]> # Copyright 2007 Kenneth Loafman <[email protected]> diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/bin/rdiffdir.1 new/duplicity-0.7.14/bin/rdiffdir.1 --- old/duplicity-0.7.13.1/bin/rdiffdir.1 2017-06-18 17:36:30.000000000 +0200 +++ new/duplicity-0.7.14/bin/rdiffdir.1 2017-08-31 14:25:19.000000000 +0200 @@ -1,4 +1,4 @@ -.TH RDIFFDIR 1 "June 18, 2017" "Version 0.7.13.1" "User Manuals" \" -*- nroff -*- +.TH RDIFFDIR 1 "August 31, 2017" "Version 0.7.14" "User Manuals" \" -*- nroff -*- .\" disable justification (adjust text to left margin only) .\" command line examples stay readable through that .ad l diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/duplicity/backends/b2backend.py new/duplicity-0.7.14/duplicity/backends/b2backend.py --- old/duplicity-0.7.13.1/duplicity/backends/b2backend.py 2017-02-08 17:49:12.000000000 +0100 +++ new/duplicity-0.7.14/duplicity/backends/b2backend.py 2017-08-29 18:03:14.000000000 +0200 @@ -222,11 +222,9 @@ if bucket_name not in bucket_names: self.create_bucket(bucket_name) else: - self.bucket_id = { - x[ - 'bucketName' - ]: x['bucketId'] for x in resp['buckets'] - }[bucket_name] + for x in resp['buckets']: + if x['bucketName'] == self.bucket_name: + self.bucket_id = x['bucketId'] def create_bucket(self, bucket_name): """ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/duplicity/backends/giobackend.py new/duplicity-0.7.14/duplicity/backends/giobackend.py --- old/duplicity-0.7.13.1/duplicity/backends/giobackend.py 2016-01-29 12:38:13.000000000 +0100 +++ new/duplicity-0.7.14/duplicity/backends/giobackend.py 2017-08-15 22:17:08.000000000 +0200 @@ -116,8 +116,11 @@ def __copy_file(self, source, target): from gi.repository import Gio # @UnresolvedImport + # Don't pass NOFOLLOW_SYMLINKS here. Some backends (e.g. google-drive:) + # use symlinks internally for all files. In the normal course of + # events, we never deal with symlinks anyway, just tarballs. source.copy(target, - Gio.FileCopyFlags.OVERWRITE | Gio.FileCopyFlags.NOFOLLOW_SYMLINKS, + Gio.FileCopyFlags.OVERWRITE, None, self.__copy_progress, None) def _error_code(self, operation, e): @@ -138,34 +141,38 @@ def _put(self, source_path, remote_filename): from gi.repository import Gio # @UnresolvedImport source_file = Gio.File.new_for_path(source_path.name) - target_file = self.remote_file.get_child(remote_filename) + target_file = self.remote_file.get_child_for_display_name(remote_filename) self.__copy_file(source_file, target_file) def _get(self, filename, local_path): from gi.repository import Gio # @UnresolvedImport - source_file = self.remote_file.get_child(filename) + source_file = self.remote_file.get_child_for_display_name(filename) target_file = Gio.File.new_for_path(local_path.name) self.__copy_file(source_file, target_file) def _list(self): from gi.repository import Gio # @UnresolvedImport files = [] - enum = self.remote_file.enumerate_children(Gio.FILE_ATTRIBUTE_STANDARD_NAME, - Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS, + # We grab display name, rather than file name because some backends + # (e.g. google-drive:) use filesystem-specific IDs as file names and + # only expose the "normal" name as display names. We need the display + # name, because we try to parse them. + enum = self.remote_file.enumerate_children(Gio.FILE_ATTRIBUTE_STANDARD_DISPLAY_NAME, + Gio.FileQueryInfoFlags.NONE, None) info = enum.next_file(None) while info: - files.append(info.get_name()) + files.append(info.get_display_name()) info = enum.next_file(None) return files def _delete(self, filename): - target_file = self.remote_file.get_child(filename) + target_file = self.remote_file.get_child_for_display_name(filename) target_file.delete(None) def _query(self, filename): from gi.repository import Gio # @UnresolvedImport - target_file = self.remote_file.get_child(filename) + target_file = self.remote_file.get_child_for_display_name(filename) info = target_file.query_info(Gio.FILE_ATTRIBUTE_STANDARD_SIZE, Gio.FileQueryInfoFlags.NONE, None) return {'size': info.get_size()} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/duplicity/backends/megabackend.py new/duplicity-0.7.14/duplicity/backends/megabackend.py --- old/duplicity-0.7.13.1/duplicity/backends/megabackend.py 2015-02-02 14:12:11.000000000 +0100 +++ new/duplicity-0.7.14/duplicity/backends/megabackend.py 2017-08-29 17:42:28.000000000 +0200 @@ -1,9 +1,7 @@ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # -# Copyright 2011 Carlos Abalde <[email protected]> -# for gdocsbackend.py on which megabackend.py is based on -# -# Copyright 2013 Christian Kornacker <[email protected]> +# Copyright 2017 Tomas Vondra (Launchpad id: tomas-v) +# Copyright 2017 Kenneth Loafman <[email protected]> # # This file is part of duplicity. # @@ -25,6 +23,9 @@ from duplicity import log from duplicity.errors import BackendException +import os +import subprocess + class MegaBackend(duplicity.backend.Backend): """Connect to remote store using Mega.co.nz API""" @@ -32,95 +33,147 @@ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) + # ensure all the necessary megatools binaries exist + self._check_binary_exists('megals') + self._check_binary_exists('megamkdir') + self._check_binary_exists('megaget') + self._check_binary_exists('megaput') + self._check_binary_exists('megarm') + + # store some basic info + self._hostname = parsed_url.hostname + + if parsed_url.password is None: + self._megarc = os.getenv('HOME') + '/.megarc' + else: + self._megarc = False + self._username = parsed_url.username + self._password = self.get_password() + + # remote folder (Can we assume /Root prefix?) + self._root = '/Root' + self._folder = self._root + '/' + parsed_url.path[1:] + + # make sure the remote folder exists (the whole path) + self._makedir_recursive(parsed_url.path[1:].split('/')) + + def _check_binary_exists(self, cmd): + 'checks that a specified command exists in the current path' + try: - from mega import Mega - except ImportError: - raise BackendException('Mega.co.nz backend requires Mega.co.nz APIs Python Module' - '(see https://github.com/richardasaurus/mega.py).') - - # Setup client instance. - self.client = Mega() - self.client.domain = parsed_url.hostname - self.__authorize(parsed_url.username, self.get_password()) - - # Fetch destination folder entry (and crete hierarchy if required). - folder_names = parsed_url.path[1:].split('/') - files = self.client.get_files() - - parent_folder = self.client.root_id - for folder_name in folder_names: - entries = self.__filter_entries(files, parent_folder, folder_name, 'folder') - if len(entries): - # use first matching folder as new parent - parent_folder = entries.keys()[0] - else: - # create subfolder if folder doesn't exist and use its handle as parent - folder_node = self.client.create_folder(folder_name, parent_folder) - parent_folder = self.client.get_id_from_obj(folder_node) - # update filelist after creating new folder - files = self.client.get_files() + # ignore the output, we only need the return code + subprocess.check_output(['which', cmd]) + except Exception as e: + raise BackendException("command '%s' not found, make sure megatools are installed" % (cmd,)) + + def _makedir(self, path): + 'creates a remote directory' + + if self._megarc: + cmd = ['megamkdir', '--config', self._megarc, path] + else: + cmd = ['megamkdir', '-u', self._username, '-p', self._password, path] - self.folder = parent_folder + self.subprocess_popen(cmd) + + def _makedir_recursive(self, path): + 'creates a remote directory (recursively the whole path), ingores errors' + + print ("mkdir: %s" % ('/'.join(path),)) + + p = self._root + + for folder in path: + p = p + '/' + folder + try: + self._make_dir(p) + except: + pass def _put(self, source_path, remote_filename): + 'uploads file to Mega (deletes it first, to ensure it does not exist)' + try: - self._delete(remote_filename) + self.delete(remote_filename) except Exception: pass - self.client.upload(source_path.get_canonical(), self.folder, dest_filename=remote_filename) + + self.upload(local_file=source_path.get_canonical(), remote_file=remote_filename) def _get(self, remote_filename, local_path): - files = self.client.get_files() - entries = self.__filter_entries(files, self.folder, remote_filename, 'file') - if len(entries): - # get first matching remote file - entry = entries.keys()[0] - self.client.download((entry, entries[entry]), dest_filename=local_path.name) - else: - raise BackendException("Failed to find file '%s' in remote folder '%s'" - % (remote_filename, self.__get_node_name(self.folder)), - code=log.ErrorCode.backend_not_found) + 'downloads file from Mega' + + self.download(remote_file=remote_filename, local_file=local_path.name) def _list(self): - entries = self.client.get_files_in_node(self.folder) - return [self.client.get_name_from_file({entry: entries[entry]}) for entry in entries] + 'list files in the backup folder' + + return self.folder_contents(files_only=True) def _delete(self, filename): - files = self.client.get_files() - entries = self.__filter_entries(files, self.folder, filename, 'file') - if len(entries): - self.client.destroy(entries.keys()[0]) - else: - raise BackendException("Failed to find file '%s' in remote folder '%s'" - % (filename, self.__get_node_name(self.folder)), - code=log.ErrorCode.backend_not_found) - - def __get_node_name(self, handle): - """get node name from public handle""" - files = self.client.get_files() - return self.client.get_name_from_file({handle: files[handle]}) - - def __authorize(self, email, password): - self.client.login(email, password) - - def __filter_entries(self, entries, parent_id=None, title=None, type=None): - result = {} - type_map = {'folder': 1, 'file': 0} + 'deletes remote ' - for k, v in entries.items(): - try: - if parent_id is not None: - assert(v['p'] == parent_id) - if title is not None: - assert(v['a']['n'] == title) - if type is not None: - assert(v['t'] == type_map[type]) - except AssertionError: - continue + self.delete(remote_file=filename) + + def folder_contents(self, files_only=False): + 'lists contents of a folder, optionally ignoring subdirectories' + + print ("megals: %s" % (self._folder,)) + + if self._megarc: + cmd = ['megals', '--config', self._megarc, self._folder] + else: + cmd = ['megals', '-u', self._username, '-p', self._password, self._folder] + + files = subprocess.check_output(cmd) + files = files.strip().split('\n') + + # remove the folder name, including the path separator + files = [f[len(self._folder) + 1:] for f in files] + + # optionally ignore entries containing path separator (i.e. not files) + if files_only: + files = [f for f in files if '/' not in f] + + return files + + def download(self, remote_file, local_file): + + print ("megaget: %s" % (remote_file,)) + + if self._megarc: + cmd = ['megaget', '--config', self._megarc, '--no-progress', + '--path', local_file, self._folder + '/' + remote_file] + else: + cmd = ['megaget', '-u', self._username, '-p', self._password, '--no-progress', + '--path', local_file, self._folder + '/' + remote_file] + + self.subprocess_popen(cmd) + + def upload(self, local_file, remote_file): + + print ("megaput: %s" % (remote_file,)) + + if self._megarc: + cmd = ['megaput', '--config', self._megarc, '--no-progress', + '--path', self._folder + '/' + remote_file, local_file] + else: + cmd = ['megaput', '-u', self._username, '-p', self._password, '--no-progress', + '--path', self._folder + '/' + remote_file, local_file] + + self.subprocess_popen(cmd) + + def delete(self, remote_file): + + print ("megarm: %s" % (remote_file,)) + + if self._megarc: + cmd = ['megarm', '--config', self._megarc, self._folder + '/' + remote_file] + else: + cmd = ['megarm', '-u', self._username, '-p', self._password, self._folder + '/' + remote_file] - result.update({k: v}) + self.subprocess_popen(cmd) - return result duplicity.backend.register_backend('mega', MegaBackend) duplicity.backend.uses_netloc.extend(['mega']) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/duplicity/backends/webdavbackend.py new/duplicity-0.7.14/duplicity/backends/webdavbackend.py --- old/duplicity-0.7.13.1/duplicity/backends/webdavbackend.py 2016-07-02 14:44:03.000000000 +0200 +++ new/duplicity-0.7.14/duplicity/backends/webdavbackend.py 2017-08-24 22:20:05.000000000 +0200 @@ -255,11 +255,11 @@ try: return self.get_kerberos_authorization() except ImportError: - log.warn(_("python-kerberos needed to use kerberos \ + log.Warn(_("python-kerberos needed to use kerberos \ authorization, falling back to basic auth.")) return self.get_basic_authorization() except Exception as e: - log.warn(_("Kerberos authorization failed: %s.\ + log.Warn(_("Kerberos authorization failed: %s.\ Falling back to basic auth.") % e) return self.get_basic_authorization() elif token.lower() == 'basic': diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/duplicity/collections.py new/duplicity-0.7.14/duplicity/collections.py --- old/duplicity-0.7.13.1/duplicity/collections.py 2016-07-02 14:44:03.000000000 +0200 +++ new/duplicity-0.7.14/duplicity/collections.py 2017-08-30 17:04:15.000000000 +0200 @@ -46,7 +46,7 @@ """ Backup set - the backup information produced by one session """ - def __init__(self, backend): + def __init__(self, backend, action): """ Initialize new backup set, only backend is required at first """ @@ -61,6 +61,7 @@ self.partial = False # true if a partial backup self.encrypted = False # true if an encrypted backup self.files_changed = [] + self.action = action def is_complete(self): """ @@ -136,7 +137,11 @@ remote_filename) self.remote_manifest_name = remote_filename - for local_filename in globals.archive_dir.listdir(): + if self.action not in ["collection-status"]: + local_filename_list = globals.archive_dir.listdir() + else: + local_filename_list = [] + for local_filename in local_filename_list: pr = file_naming.parse(local_filename) if (pr and pr.manifest and pr.type == self.type and pr.time == self.time and @@ -159,7 +164,11 @@ except Exception: log.Debug(_("BackupSet.delete: missing %s") % [util.ufn(f) for f in rfn]) pass - for lfn in globals.archive_dir.listdir(): + if self.action not in ["collection-status"]: + local_filename_list = globals.archive_dir.listdir() + else: + local_filename_list = [] + for lfn in local_filename_list: pr = file_naming.parse(lfn) if (pr and pr.time == self.time and pr.start_time == self.start_time and @@ -228,17 +237,12 @@ Return manifest by reading remote manifest on backend """ assert self.remote_manifest_name - # Following by MDR. Should catch if remote encrypted with - # public key w/o secret key try: manifest_buffer = self.backend.get_data(self.remote_manifest_name) except GPGError as message: - # TODO: We check for gpg v1 and v2 messages, should be an error code - if ("secret key not available" in message.args[0] or - "No secret key" in message.args[0]): - return None - else: - raise + log.Error(_("Error processing remote manifest (%s): %s") % + (self.remote_manifest_name, str(message))) + return None log.Info(_("Processing remote manifest %s (%s)") % (self.remote_manifest_name, len(manifest_buffer))) return manifest.Manifest().from_string(manifest_buffer) @@ -265,7 +269,7 @@ # when specifically asked for a list of remote filenames, we # should not include it. pr = file_naming.parse(self.remote_manifest_name) - if not pr or not pr.partial: + if pr and not pr.partial: volume_filenames.append(self.remote_manifest_name) return volume_filenames @@ -582,12 +586,13 @@ """ Hold information about available chains and sets """ - def __init__(self, backend, archive_dir): + def __init__(self, backend, archive_dir, action): """ Make new object. Does not set values """ self.backend = backend self.archive_dir = archive_dir + self.action = action # Will hold (signature chain, backup chain) pair of active # (most recent) chains @@ -691,7 +696,10 @@ len(backend_filename_list)) # get local filename list - local_filename_list = self.archive_dir.listdir() + if self.action not in ["collection-status"]: + local_filename_list = self.archive_dir.listdir() + else: + local_filename_list = [] log.Debug(ngettext("%d file exists in cache", "%d files exist in cache", len(local_filename_list)) % @@ -826,7 +834,7 @@ break else: log.Debug(_("File %s is not part of a known set; creating new set") % (util.ufn(filename),)) - new_set = BackupSet(self.backend) + new_set = BackupSet(self.backend, self.action) if new_set.add_filename(filename): sets.append(new_set) else: @@ -888,7 +896,10 @@ if filelist is not None: return filelist elif local: - return self.archive_dir.listdir() + if self.action not in ["collection-status"]: + return self.archive_dir.listdir() + else: + return [] else: return self.backend.list() @@ -1174,7 +1185,7 @@ Returns time line of specified file changed """ # quick fix to spaces in filepath - modified_filepath = "" + modified_filepath = filepath if " " in filepath: modified_filepath = '"' + filepath.replace(" ", r"\x20") + '"' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/duplicity/commandline.py new/duplicity-0.7.14/duplicity/commandline.py --- old/duplicity-0.7.13.1/duplicity/commandline.py 2017-01-21 14:01:05.000000000 +0100 +++ new/duplicity-0.7.14/duplicity/commandline.py 2017-08-31 13:14:19.000000000 +0200 @@ -383,7 +383,7 @@ # --archive-dir <path> parser.add_option("--file-to-restore", "-r", action="callback", type="file", metavar=_("path"), dest="restore_dir", - callback=lambda o, s, v, p: setattr(p.values, "restore_dir", v.rstrip('/'))) + callback=lambda o, s, v, p: setattr(p.values, "restore_dir", v.strip('/'))) # Used to confirm certain destructive operations like deleting old files. parser.add_option("--force", action="store_true") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/duplicity/globals.py new/duplicity-0.7.14/duplicity/globals.py --- old/duplicity-0.7.13.1/duplicity/globals.py 2017-06-18 17:36:30.000000000 +0200 +++ new/duplicity-0.7.14/duplicity/globals.py 2017-08-31 14:25:19.000000000 +0200 @@ -26,7 +26,7 @@ # The current version of duplicity -version = "0.7.13.1" +version = "0.7.14" # Prefix for all files (appended before type-specific prefixes) file_prefix = "" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/duplicity/gpg.py new/duplicity-0.7.14/duplicity/gpg.py --- old/duplicity-0.7.13.1/duplicity/gpg.py 2017-04-28 18:07:24.000000000 +0200 +++ new/duplicity-0.7.14/duplicity/gpg.py 2017-08-28 17:27:05.000000000 +0200 @@ -91,7 +91,7 @@ self.gpg_version = self.get_gpg_version(globals.gpg_binary) - _version_re = re.compile(r'^gpg.*\(GnuPG(?:/MacGPG2)?\) (?P<maj>[0-9]+)\.(?P<min>[0-9]+)\.(?P<bug>[0-9]+)$') + _version_re = re.compile(r'^gpg.*\(GnuPG(?:/MacGPG2)?\) (?P<maj>[0-9]+)\.(?P<min>[0-9]+)\.(?P<bug>[0-9]+)(-.+)?$') def get_gpg_version(self, binary): gpg = gpginterface.GnuPG() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/duplicity/manifest.py new/duplicity-0.7.14/duplicity/manifest.py --- old/duplicity-0.7.13.1/duplicity/manifest.py 2016-07-02 14:44:03.000000000 +0200 +++ new/duplicity-0.7.14/duplicity/manifest.py 2017-08-30 21:51:07.000000000 +0200 @@ -205,7 +205,11 @@ return (fileinfo[0], ''.join(fileinfo[1:])) self.files_changed = list(map(parse_fileinfo, match.group(3).split('\n'))) - assert filecount == len(self.files_changed) + + if filecount != len(self.files_changed): + log.Error(_("Manifest file '%s' is corrupt: File count says %d, File list contains %d" % + (self.fh.base if self.fh else "", filecount, len(self.files_changed)))) + self.corrupt_filelist = True highest_vol = 0 latest_vol = 0 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/setup.py new/duplicity-0.7.14/setup.py --- old/duplicity-0.7.13.1/setup.py 2017-06-18 17:36:30.000000000 +0200 +++ new/duplicity-0.7.14/setup.py 2017-08-31 14:25:19.000000000 +0200 @@ -28,7 +28,7 @@ from setuptools.command.sdist import sdist from distutils.command.build_scripts import build_scripts -version_string = "0.7.13.1" +version_string = "0.7.14" if sys.version_info[:2] < (2, 6) or sys.version_info[:2] > (2, 7): print("Sorry, duplicity requires version 2.6 or 2.7 of python.") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/testing/functional/__init__.py new/duplicity-0.7.14/testing/functional/__init__.py --- old/duplicity-0.7.13.1/testing/functional/__init__.py 2017-04-18 23:06:23.000000000 +0200 +++ new/duplicity-0.7.14/testing/functional/__init__.py 2017-07-18 13:52:57.000000000 +0200 @@ -120,14 +120,14 @@ if fail: self.assertEqual(30, return_val) elif return_val: -# print >>sys.stderr, "\n...command:", cmdline -# print >>sys.stderr, "...cwd:", os.getcwd() -# print >>sys.stderr, "...output:" -# for line in lines: -# line = line.rstrip() -# if line: -# print >>sys.stderr, line -# print >>sys.stderr, "...return_val:", return_val + print >>sys.stderr, "\n...command:", cmdline + print >>sys.stderr, "...cwd:", os.getcwd() + print >>sys.stderr, "...output:" + for line in lines: + line = line.rstrip() + if line: + print >>sys.stderr, line + print >>sys.stderr, "...return_val:", return_val raise CmdError(return_val) def backup(self, type, input_dir, options=[], **kwargs): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/testing/unit/test_collections.py new/duplicity-0.7.14/testing/unit/test_collections.py --- old/duplicity-0.7.13.1/testing/unit/test_collections.py 2015-02-02 14:18:25.000000000 +0100 +++ new/duplicity-0.7.14/testing/unit/test_collections.py 2017-08-06 18:25:09.000000000 +0200 @@ -97,7 +97,7 @@ def test_backup_chains(self): """Test basic backup chain construction""" random.shuffle(filename_list1) - cs = collections.CollectionsStatus(None, globals.archive_dir) + cs = collections.CollectionsStatus(None, globals.archive_dir, "full") chains, orphaned, incomplete = cs.get_backup_chains(filename_list1) # @UnusedVariable if len(chains) != 1 or len(orphaned) != 0: print chains @@ -118,7 +118,7 @@ assert cs.matched_chain_pair[0].end_time == 1029826800 assert len(cs.all_backup_chains) == 1, cs.all_backup_chains - cs = collections.CollectionsStatus(self.real_backend, globals.archive_dir).set_values() + cs = collections.CollectionsStatus(self.real_backend, globals.archive_dir, "full").set_values() check_cs(cs) assert cs.matched_chain_pair[0].islocal() @@ -131,13 +131,13 @@ def test_sig_chains(self): """Test making signature chains from filename list""" - cs = collections.CollectionsStatus(None, globals.archive_dir) + cs = collections.CollectionsStatus(None, globals.archive_dir, "full") chains, orphaned_paths = cs.get_signature_chains(local=1) self.sig_chains_helper(chains, orphaned_paths) def test_sig_chains2(self): """Test making signature chains from filename list on backend""" - cs = collections.CollectionsStatus(self.archive_dir_backend, globals.archive_dir) + cs = collections.CollectionsStatus(self.archive_dir_backend, globals.archive_dir, "full") chains, orphaned_paths = cs.get_signature_chains(local=None) self.sig_chains_helper(chains, orphaned_paths) @@ -194,7 +194,7 @@ p = self.output_dir.append(filename) p.touch() - cs = collections.CollectionsStatus(self.output_dir_backend, globals.archive_dir) + cs = collections.CollectionsStatus(self.output_dir_backend, globals.archive_dir, "full") cs.set_values() return cs diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.13.1/testing/unit/test_manifest.py new/duplicity-0.7.14/testing/unit/test_manifest.py --- old/duplicity-0.7.13.1/testing/unit/test_manifest.py 2015-05-08 14:10:32.000000000 +0200 +++ new/duplicity-0.7.14/testing/unit/test_manifest.py 2017-08-31 12:21:03.000000000 +0200 @@ -19,11 +19,15 @@ # along with duplicity; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +from StringIO import StringIO +import re +import sys import types import unittest from duplicity import manifest from duplicity import path + from . import UnitTestCase @@ -92,13 +96,38 @@ m.set_files_changed_info([]) s = m.to_string() - # print "---------\n%s\n---------" % s assert s.lower().startswith("hostname") assert s.endswith("\n") m2 = manifest.Manifest().from_string(s) assert m == m2 + def test_corrupt_filelist(self): + vi1 = manifest.VolumeInfo() + vi1.set_info(3, ("hello",), None, (), None) + vi2 = manifest.VolumeInfo() + vi2.set_info(4, ("goodbye", "there"), None, ("aoeusht",), None) + vi3 = manifest.VolumeInfo() + vi3.set_info(34, (), None, (), None) + m = manifest.Manifest() + for vi in [vi1, vi2, vi3]: + m.add_volume_info(vi) + + self.set_global('local_path', path.Path("Foobar")) + m.set_dirinfo() + m.set_files_changed_info([ + ('one', 'new'), + ('two', 'changed'), + ('three', 'new'), + ]) + + # build manifest string + s = m.to_string() + + # make filecount higher than files in list + s2 = re.sub('Filelist 3', 'Filelist 5', s) + m2 = manifest.Manifest().from_string(s2) + assert hasattr(m2, 'corrupt_filelist') if __name__ == "__main__": unittest.main()
