Hello community, here is the log from the commit of package duplicity for openSUSE:Factory checked in at 2017-12-19 10:46:44 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/duplicity (Old) and /work/SRC/openSUSE:Factory/.duplicity.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "duplicity" Tue Dec 19 10:46:44 2017 rev:38 rq:557188 version:0.7.15 Changes: -------- --- /work/SRC/openSUSE:Factory/duplicity/duplicity.changes 2017-09-07 22:10:27.502279431 +0200 +++ /work/SRC/openSUSE:Factory/.duplicity.new/duplicity.changes 2017-12-19 10:46:47.387232195 +0100 @@ -1,0 +2,8 @@ +Thu Dec 14 22:03:11 UTC 2017 - [email protected] + +- update to 0.7.15 + * fixed several issues + (for upstream changes see + http://duplicity.nongnu.org/CHANGELOG + +------------------------------------------------------------------- Old: ---- duplicity-0.7.14.tar.gz New: ---- duplicity-0.7.15.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ duplicity.spec ++++++ --- /var/tmp/diff_new_pack.xOGc8k/_old 2017-12-19 10:46:48.519177552 +0100 +++ /var/tmp/diff_new_pack.xOGc8k/_new 2017-12-19 10:46:48.519177552 +0100 @@ -19,7 +19,7 @@ %{!?python_sitelib: %global python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} %{!?python_sitearch: %global python_sitearch %(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} Name: duplicity -Version: 0.7.14 +Version: 0.7.15 Release: 0 Summary: Encrypted bandwidth-efficient backup using the rsync algorithm License: GPL-3.0+ ++++++ duplicity-0.7.14.tar.gz -> duplicity-0.7.15.tar.gz ++++++ ++++ 1761 lines of diff (skipped) ++++ retrying with extended exclude list diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/CHANGELOG new/duplicity-0.7.15/CHANGELOG --- old/duplicity-0.7.14/CHANGELOG 2017-08-31 14:06:41.000000000 +0200 +++ new/duplicity-0.7.15/CHANGELOG 2017-11-13 16:47:44.000000000 +0100 @@ -1,3 +1,43 @@ +New in v0.7.15 (2017/11/13) +--------------------------- +* Fixed bug introduced in new megabackend.py where process_commandline() + takes a string not a list. Now it takes both. +* Updated web page for new megabackend requirements. +* Patched in lp:~mterry/duplicity/more-decode-issues + - Here's some fixes for another couple UnicodeDecodeErrors. + - The duplicity/dup_time.py fixes when a user passes a utf8 date string (or a string with bogus + utf8 characters, but they have to really try to do that). This is bug 1334436. + - The bin/duplicity change from str(e) to util.uexc(e) fixes bug 1324188. + - The rest of the changes (util.exception_traceback and bin/duplicity changes to use it) are to + make the printing of exceptions prettier. Without this, if you see a French exception, you see + "accept\xe9es" instead of "acceptées". + - You can test all of these changes in one simple line: + $ LANGUAGE=fr duplicity remove-older-than $'accept\xffées' +* Fix backend.py to allow string, list, and tuple types to support megabackend.py. +* Fixed bug #1715650 with patch from Mattheww S + - Fix to make duplicity attempt a get first, then create, a container + in order to support container ACLs. +* Fixed bug #1714663 "Volume signed by XXXXXXXXXXXXXXXX, not XXXXXXXX" + - Normalized comparison length to min length of compared keys before comparison + - Avoids comparing mix of short, long, or fingerprint size keys. +* Merged in lp:~mterry/duplicity/rename-dep + - Make rename command a dependency for LP build +* Fixed bug #1654756 with new b2backend.py module from Vincent Rouille + - Faster (big files are uploaded in chunks) + - Added upload progress reporting support +* Fixed bug #1448094 with patch from Tomáš Zvala + - Don't log incremental deletes for chains that have no incrementals +* Fixed bug #1724144 "--gpg-options unused with some commands" + - Add --gpg-options to get version run command +* Fixed bug #1720159 - Cannot allocate memory with large manifest file since 0.7.03 + - filelist is not read if --file-changed option in collection-status not present + - This will keep memory usage lower in non collection-status operations +* Fixed bug #1723890 with patch from Killian Lackhove + - Fixes error handling in pydrivebackend.py +* Fixed bug #1730902 GPG Error Handling + - use util.ufn() not str() to handle encoding + + New in v0.7.14 (2017/08/31) --------------------------- * Merged in lp:~dawgfoto/duplicity/skip_sync_collection_status diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/Changelog.GNU new/duplicity-0.7.15/Changelog.GNU --- old/duplicity-0.7.14/Changelog.GNU 2017-08-31 14:06:14.000000000 +0200 +++ new/duplicity-0.7.15/Changelog.GNU 2017-11-13 16:48:28.000000000 +0100 @@ -1,3 +1,73 @@ +2017-11-09 Kenneth Loafman <[email protected]> + + * Prep for 0.7.15 + +2017-11-09 Kenneth Loafman <[email protected]> + + * Fixed bug #1730902 GPG Error Handling + - use util.ufn() not str() to handle encoding + +2017-11-01 Kenneth Loafman <[email protected]> + + * Fixed bug #1723890 with patch from Killian Lackhove + - Fixes error handling in pydrivebackend.py + +2017-10-31 Kenneth Loafman <[email protected]> + + * Fixed bug #1720159 - Cannot allocate memory with large manifest file since 0.7.03 + - filelist is not read if --file-changed option in collection-status not present + - This will keep memory usage lower in non collection-status operations + +2017-10-26 Kenneth Loafman <[email protected]> + + * Fixed bug #1448094 with patch from Tomáš Zvala + - Don't log incremental deletes for chains that have no incrementals + * Fixed bug #1724144 "--gpg-options unused with some commands" + - Add --gpg-options to get version run command + +2017-10-16 Kenneth Loafman <[email protected]> + + * Fixed bug #1654756 with new b2backend.py module from Vincent Rouille + - Faster (big files are uploaded in chunks) + - Added upload progress reporting support + +2017-10-12 Kenneth Loafman <[email protected]> + + * Merged in lp:~mterry/duplicity/rename-dep + - Make rename command a dependency for LP build + +2017-09-22 Kenneth Loafman <[email protected]> + + * Fixed bug #1714663 "Volume signed by XXXXXXXXXXXXXXXX, not XXXXXXXX" + - Normalized comparison length to min length of compared keys before comparison + - Avoids comparing mix of short, long, or fingerprint size keys. + +2017-09-13 Kenneth Loafman <[email protected]> + + * Fixed bug #1715650 with patch from Mattheww S + - Fix to make duplicity attempt a get first, then create, a container + in order to support container ACLs. + +2017-09-07 Kenneth Loafman <[email protected]> + + * Merged in lp:~mterry/duplicity/more-decode-issues + - Here's some fixes for another couple UnicodeDecodeErrors. + - The duplicity/dup_time.py fixes when a user passes a utf8 date string (or a string with bogus + utf8 characters, but they have to really try to do that). This is bug 1334436. + - The bin/duplicity change from str(e) to util.uexc(e) fixes bug 1324188. + - The rest of the changes (util.exception_traceback and bin/duplicity changes to use it) are to + make the printing of exceptions prettier. Without this, if you see a French exception, you see + "accept\xe9es" instead of "acceptées". + - You can test all of these changes in one simple line: + $ LANGUAGE=fr duplicity remove-older-than $'accept\xffées' + * Fix backend.py to allow string, list, and tuple types to support megabackend.py. + +2017-09-06 Kenneth Loafman <[email protected]> + + * Fixed bug introduced in new megabackend.py where process_commandline() + takes a string not a list. Now it takes both. + * Updated web page for new megabackend requirements. + 2017-08-31 Kenneth Loafman <[email protected]> * Fixed bug #1538333 Assertion error in manifest.py: assert filecount == ... diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/bin/duplicity new/duplicity-0.7.15/bin/duplicity --- old/duplicity-0.7.14/bin/duplicity 2017-08-31 14:25:19.000000000 +0200 +++ new/duplicity-0.7.15/bin/duplicity 2017-11-13 16:56:58.000000000 +0100 @@ -2,7 +2,7 @@ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- # # duplicity -- Encrypted bandwidth efficient backup -# Version 0.7.14 released August 31, 2017 +# Version 0.7.15 released November 13, 2017 # # Copyright 2002 Ben Escoto <[email protected]> # Copyright 2007 Kenneth Loafman <[email protected]> @@ -841,9 +841,13 @@ def check_signature(): """Thunk run when closing volume file""" actual_sig = fileobj.fileobj.get_signature() - if actual_sig != globals.gpg_profile.sign_key: + actual_sig = "None" if actual_sig is None else actual_sig + sign_key = globals.gpg_profile.sign_key + sign_key = "None" if sign_key is None else sign_key + ofs = -min(len(actual_sig), len(sign_key)) + if actual_sig[ofs:] != sign_key[ofs:]: log.FatalError(_("Volume was signed by key %s, not %s") % - (actual_sig, globals.gpg_profile.sign_key), + (actual_sig[ofs:], sign_key[ofs:]), log.ErrorCode.unsigned_volume) fileobj.addhook(check_signature) @@ -969,6 +973,13 @@ "manually purge the repository.")) chainlist = col_stats.get_chains_older_than(globals.remove_time) + + if globals.remove_all_inc_of_but_n_full_mode: + # ignore chains without incremental backups: + chainlist = list(x for x in chainlist if + (isinstance(x, collections.SignatureChain) and x.inclist) or + (isinstance(x, collections.BackupChain) and x.incset_list)) + if not chainlist: log.Notice(_("No old backup sets found, nothing deleted.")) return @@ -1273,7 +1284,7 @@ log Python, duplicity, and system versions """ log.Log(u'=' * 80, verbosity) - log.Log(u"duplicity 0.7.14 (August 31, 2017)", verbosity) + log.Log(u"duplicity 0.7.15 (November 13, 2017)", verbosity) log.Log(u"Args: %s" % util.ufn(' '.join(sys.argv)), verbosity) log.Log(u' '.join(platform.uname()), verbosity) log.Log(u"%s %s" % (sys.executable or sys.platform, sys.version), verbosity) @@ -1574,7 +1585,7 @@ # default. But do with sufficient verbosity. util.release_lockfile() log.Info(_("GPG error detail: %s") - % (u''.join(traceback.format_exception(*sys.exc_info())))) + % util.exception_traceback()) log.FatalError(u"%s: %s" % (e.__class__.__name__, e.args[0]), log.ErrorCode.gpg_failed, e.__class__.__name__) @@ -1584,7 +1595,7 @@ # For user errors, don't show an ugly stack trace by # default. But do with sufficient verbosity. log.Info(_("User error detail: %s") - % (u''.join(traceback.format_exception(*sys.exc_info())))) + % util.exception_traceback()) log.FatalError(u"%s: %s" % (e.__class__.__name__, util.uexc(e)), log.ErrorCode.user_error, e.__class__.__name__) @@ -1594,19 +1605,19 @@ # For backend errors, don't show an ugly stack trace by # default. But do with sufficient verbosity. log.Info(_("Backend error detail: %s") - % (u''.join(traceback.format_exception(*sys.exc_info())))) + % util.exception_traceback()) log.FatalError(u"%s: %s" % (e.__class__.__name__, util.uexc(e)), log.ErrorCode.user_error, e.__class__.__name__) except Exception as e: util.release_lockfile() - if "Forced assertion for testing" in str(e): + if "Forced assertion for testing" in util.uexc(e): log.FatalError(u"%s: %s" % (e.__class__.__name__, util.uexc(e)), log.ErrorCode.exception, e.__class__.__name__) else: # Traceback and that mess - log.FatalError(u''.join(traceback.format_exception(*sys.exc_info())), + log.FatalError(util.exception_traceback(), log.ErrorCode.exception, e.__class__.__name__) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/bin/duplicity.1 new/duplicity-0.7.15/bin/duplicity.1 --- old/duplicity-0.7.14/bin/duplicity.1 2017-08-31 14:25:19.000000000 +0200 +++ new/duplicity-0.7.15/bin/duplicity.1 2017-11-13 16:56:58.000000000 +0100 @@ -1,4 +1,4 @@ -.TH DUPLICITY 1 "August 31, 2017" "Version 0.7.14" "User Manuals" \" -*- nroff -*- +.TH DUPLICITY 1 "November 13, 2017" "Version 0.7.15" "User Manuals" \" -*- nroff -*- .\" disable justification (adjust text to left margin only) .\" command line examples stay readable through that .ad l @@ -2137,8 +2137,8 @@ - http://lftp.yar.ru/ .TP .BR "mega backend" " (mega.co.nz)" -.B Python library for mega API -- https://github.com/ckornacker/mega.py, ubuntu ppa - ppa:ckornacker/backup +.B megatools client +- https://github.com/megous/megatools .TP .BR "multi backend" .B Multi -- store to more than one backend diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/bin/rdiffdir new/duplicity-0.7.15/bin/rdiffdir --- old/duplicity-0.7.14/bin/rdiffdir 2017-08-31 14:25:19.000000000 +0200 +++ new/duplicity-0.7.15/bin/rdiffdir 2017-11-13 16:56:58.000000000 +0100 @@ -1,6 +1,6 @@ #!/usr/bin/env python2 # rdiffdir -- Extend rdiff functionality to directories -# Version 0.7.14 released August 31, 2017 +# Version 0.7.15 released November 13, 2017 # # Copyright 2002 Ben Escoto <[email protected]> # Copyright 2007 Kenneth Loafman <[email protected]> diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/bin/rdiffdir.1 new/duplicity-0.7.15/bin/rdiffdir.1 --- old/duplicity-0.7.14/bin/rdiffdir.1 2017-08-31 14:25:19.000000000 +0200 +++ new/duplicity-0.7.15/bin/rdiffdir.1 2017-11-13 16:56:58.000000000 +0100 @@ -1,4 +1,4 @@ -.TH RDIFFDIR 1 "August 31, 2017" "Version 0.7.14" "User Manuals" \" -*- nroff -*- +.TH RDIFFDIR 1 "November 13, 2017" "Version 0.7.15" "User Manuals" \" -*- nroff -*- .\" disable justification (adjust text to left margin only) .\" command line examples stay readable through that .ad l diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/debian/control new/duplicity-0.7.15/debian/control --- old/duplicity-0.7.14/debian/control 2017-05-11 13:53:20.000000000 +0200 +++ new/duplicity-0.7.15/debian/control 2017-10-12 21:03:15.000000000 +0200 @@ -15,6 +15,7 @@ python-pexpect, python-setuptools, rdiff, + rename, rsync, Homepage: https://launchpad.net/duplicity Standards-Version: 3.9.5 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/duplicity/backend.py new/duplicity-0.7.15/duplicity/backend.py --- old/duplicity-0.7.14/duplicity/backend.py 2016-12-09 16:36:26.000000000 +0100 +++ new/duplicity-0.7.15/duplicity/backend.py 2017-09-07 18:46:12.000000000 +0200 @@ -450,15 +450,14 @@ else: return commandline - def __subprocess_popen(self, commandline): + def __subprocess_popen(self, args): """ For internal use. Execute the given command line, interpreted as a shell command. Returns int Exitcode, string StdOut, string StdErr """ - import shlex from subprocess import Popen, PIPE - args = shlex.split(commandline) + args[0] = util.which(args[0]) p = Popen(args, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() @@ -476,20 +475,28 @@ Raise a BackendException on failure. """ - private = self.munge_password(commandline) - log.Info(_("Reading results of '%s'") % private) - result, stdout, stderr = self.__subprocess_popen(commandline) + import shlex + + if isinstance(commandline, (types.ListType, types.TupleType)): + logstr = ' '.join(commandline) + args = commandline + else: + logstr = commandline + args = shlex.split(commandline) + + logstr = self.munge_password(logstr) + log.Info(_("Reading results of '%s'") % logstr) + + result, stdout, stderr = self.__subprocess_popen(args) if result != 0: try: - m = re.search("^\s*([\S]+)", commandline) - cmd = m.group(1) - ignores = self.popen_breaks[cmd] + ignores = self.popen_breaks[args[0]] ignores.index(result) """ ignore a predefined set of error codes """ return 0, '', '' except (KeyError, ValueError): raise BackendException("Error running '%s': returned %d, with output:\n%s" % - (private, result, stdout + '\n' + stderr)) + (logstr, result, stdout + '\n' + stderr)) return result, stdout, stderr diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/duplicity/backends/_cf_pyrax.py new/duplicity-0.7.15/duplicity/backends/_cf_pyrax.py --- old/duplicity-0.7.14/duplicity/backends/_cf_pyrax.py 2017-03-13 17:42:26.000000000 +0100 +++ new/duplicity-0.7.15/duplicity/backends/_cf_pyrax.py 2017-09-16 14:21:29.000000000 +0200 @@ -70,7 +70,25 @@ self.client_exc = pyrax.exceptions.ClientException self.nso_exc = pyrax.exceptions.NoSuchObject - self.container = pyrax.cloudfiles.create_container(container) + + # query rackspace for the specified container name + try: + self.container = pyrax.cloudfiles.get_container(container) + except pyrax.exceptions.Forbidden as e: + log.FatalError("%s : %s \n" % (e.__class__.__name__, util.uexc(e)) + + "Container may exist, but access was denied.\n" + + "If this container exists, please check its X-Container-Read/Write headers.\n" + + "Otherwise, please check your credentials and permissions.", + log.ErrorCode.backend_permission_denied) + except pyrax.exceptions.NoSuchContainer as e: + try: + self.container = pyrax.cloudfiles.create_container(container) + except pyrax.exceptions.Forbidden as e: + log.FatalError("%s : %s \n" % (e.__class__.__name__, util.uexc(e)) + + "Container does not exist, but creation was denied.\n" + + "You may be using a read-only user that can view but not create containers.\n" + + "Please check your credentials and permissions.", + log.ErrorCode.backend_permission_denied) def _error_code(self, operation, e): if isinstance(e, self.nso_exc): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/duplicity/backends/b2backend.py new/duplicity-0.7.15/duplicity/backends/b2backend.py --- old/duplicity-0.7.14/duplicity/backends/b2backend.py 2017-08-29 18:03:14.000000000 +0200 +++ new/duplicity-0.7.15/duplicity/backends/b2backend.py 2017-10-26 19:40:48.000000000 +0200 @@ -24,15 +24,22 @@ import os import hashlib -from sys import version_info import duplicity.backend from duplicity.errors import BackendException, FatalBackendException from duplicity import log +from duplicity import progress -import json -import urllib2 -import base64 + +class B2ProgressListener: + def set_total_bytes(self, total_byte_count): + self.total_byte_count = total_byte_count + + def bytes_completed(self, byte_count): + progress.report_transfer(byte_count, self.total_byte_count) + + def close(self): + pass class B2Backend(duplicity.backend.Backend): @@ -46,10 +53,21 @@ """ duplicity.backend.Backend.__init__(self, parsed_url) - # for prettier password prompt only + # Import B2 API + try: + global b2 + import b2 + import b2.api + import b2.account_info + import b2.download_dest + import b2.file_version + except ImportError: + raise BackendException('B2 backend requires B2 Python APIs (pip install b2)') + + self.service = b2.api.B2Api(b2.account_info.InMemoryAccountInfo()) self.parsed_url.hostname = 'B2' - self.account_id = parsed_url.username + account_id = parsed_url.username account_key = self.get_password() self.url_parts = [ @@ -57,322 +75,72 @@ ] if self.url_parts: self.username = self.url_parts.pop(0) - self.bucket_name = self.url_parts.pop(0) + bucket_name = self.url_parts.pop(0) else: raise BackendException("B2 requires a bucket name") - self.path = "/".join(self.url_parts) - - self.id_and_key = self.account_id + ":" + account_key - self._authorize() - self.upload_info = None + self.path = "".join([url_part + "/" for url_part in self.url_parts]) + self.service.authorize_account('production', account_id, account_key) + log.Log("B2 Backend (path= %s, bucket= %s, minimum_part_size= %s)" % + (self.path, bucket_name, self.service.account_info.get_minimum_part_size()), log.INFO) try: - self.find_or_create_bucket(self.bucket_name) - except urllib2.HTTPError: - raise FatalBackendException("Bucket cannot be created") - - def _authorize(self): - basic_auth_string = 'Basic ' + base64.b64encode(self.id_and_key) - v = version_info - headers = { - 'Authorization': basic_auth_string, - 'User-Agent': 'duplicity version $version, python %s.%s.%s' % ( - v[0], v[1], v[2]), - } - - request = urllib2.Request( - 'https://api.backblazeb2.com/b2api/v1/b2_authorize_account', - headers=headers - ) - - response = urllib2.urlopen(request) - response_data = json.loads(response.read()) - response.close() - - self.auth_token = response_data['authorizationToken'] - self.api_url = response_data['apiUrl'] - self.download_url = response_data['downloadUrl'] + self.bucket = self.service.get_bucket_by_name(bucket_name) + log.Log("Bucket found", log.INFO) + except b2.exception.NonExistentBucket: + try: + log.Log("Bucket not found, creating one", log.INFO) + self.bucket = self.service.create_bucket(bucket_name, 'allPrivate') + except: + raise FatalBackendException("Bucket cannot be created") def _get(self, remote_filename, local_path): """ Download remote_filename to local_path """ - log.Log("Getting file %s" % remote_filename, 9) - remote_filename = self.full_filename(remote_filename) - url = self.download_url + \ - '/file/' + self.bucket_name + '/' + \ - remote_filename - resp = self.get_or_post(url, None) - - to_file = open(local_path.name, 'wb') - to_file.write(resp) - to_file.close() + log.Log("Get: %s -> %s" % (self.path + remote_filename, local_path.name), log.INFO) + self.bucket.download_file_by_name(self.path + remote_filename, + b2.download_dest.DownloadDestLocalFile(local_path.name)) def _put(self, source_path, remote_filename): """ Copy source_path to remote_filename """ - log.Log("Putting file to %s" % remote_filename, 9) - self._delete(remote_filename) - digest = self.hex_sha1_of_file(source_path) - content_type = 'application/pgp-encrypted' - remote_filename = self.full_filename(remote_filename) - - info = self.get_upload_info(self.bucket_id) - url = info['uploadUrl'] - - headers = { - 'Authorization': info['authorizationToken'], - 'X-Bz-File-Name': remote_filename, - 'Content-Type': content_type, - 'X-Bz-Content-Sha1': digest, - 'Content-Length': str(os.path.getsize(source_path.name)), - } - data_file = source_path.open() - self.get_or_post(url, None, headers, data_file=data_file) + log.Log("Put: %s -> %s" % (source_path.name, self.path + remote_filename), log.INFO) + self.bucket.upload_local_file(source_path.name, self.path + remote_filename, + content_type='application/pgp-encrypted', + progress_listener=B2ProgressListener()) def _list(self): """ List files on remote server """ - log.Log("Listing files", 9) - endpoint = 'b2_list_file_names' - url = self.formatted_url(endpoint) - params = { - 'bucketId': self.bucket_id, - 'maxFileCount': 1000, - 'prefix': self.path, - } - try: - resp = self.get_or_post(url, params) - except urllib2.HTTPError: - return [] - - files = [x['fileName'].split('/')[-1] for x in resp['files'] - if os.path.dirname(x['fileName']) == self.path] - - next_file = resp['nextFileName'] - while next_file: - log.Log("There are still files, getting next list", 9) - params['startFileName'] = next_file - try: - resp = self.get_or_post(url, params) - except urllib2.HTTPError: - return files - - files += [x['fileName'].split('/')[-1] for x in resp['files'] - if os.path.dirname(x['fileName']) == self.path] - next_file = resp['nextFileName'] - - return files + return [file_version_info.file_name[len(self.path):] + for (file_version_info, folder_name) in self.bucket.ls(self.path)] def _delete(self, filename): """ Delete filename from remote server """ - log.Log("Deleting file %s" % filename, 9) - endpoint = 'b2_delete_file_version' - url = self.formatted_url(endpoint) - fileid = self.get_file_id(filename) - if fileid is None: - return - filename = self.full_filename(filename) - params = {'fileName': filename, 'fileId': fileid} - try: - self.get_or_post(url, params) - except urllib2.HTTPError as e: - if e.code == 400: - return - else: - raise e + log.Log("Delete: %s" % self.path + filename, log.INFO) + file_version_info = self.file_info(self.path + filename) + self.bucket.delete_file_version(file_version_info.id_, file_version_info.file_name) def _query(self, filename): """ Get size info of filename """ - log.Log("Querying file %s" % filename, 9) - info = self.get_file_info(filename) - if not info: - return {'size': -1} - - return {'size': info['size']} - - def _error_code(self, operation, e): - if isinstance(e, urllib2.HTTPError): - if e.code == 400: - return log.ErrorCode.bad_request - if e.code == 500: - return log.ErrorCode.backend_error - if e.code == 403: - return log.ErrorCode.backend_permission_denied - - def find_or_create_bucket(self, bucket_name): - """ - Find a bucket with name bucket_name and save its id. - If it doesn't exist, create it - """ - endpoint = 'b2_list_buckets' - url = self.formatted_url(endpoint) - - params = {'accountId': self.account_id} - resp = self.get_or_post(url, params) - - bucket_names = [x['bucketName'] for x in resp['buckets']] - - if bucket_name not in bucket_names: - self.create_bucket(bucket_name) - else: - for x in resp['buckets']: - if x['bucketName'] == self.bucket_name: - self.bucket_id = x['bucketId'] - - def create_bucket(self, bucket_name): - """ - Create a bucket with name bucket_name and save its id - """ - endpoint = 'b2_create_bucket' - url = self.formatted_url(endpoint) - params = { - 'accountId': self.account_id, - 'bucketName': bucket_name, - 'bucketType': 'allPrivate' - } - resp = self.get_or_post(url, params) - - self.bucket_id = resp['bucketId'] - - def formatted_url(self, endpoint): - """ - Return the full api endpoint from just the last part - """ - return '%s/b2api/v1/%s' % (self.api_url, endpoint) - - def get_upload_info(self, bucket_id): - """ - Get an upload url for a bucket - """ - if self.upload_info is None: - endpoint = 'b2_get_upload_url' - url = self.formatted_url(endpoint) - self.upload_info = self.get_or_post(url, {'bucketId': bucket_id}) - return self.upload_info - - def get_or_post(self, url, data, headers=None, data_file=None): - """ - Sends the request, either get or post. - If data and data_file are None, send a get request. - data_file takes precedence over data. - If headers are not supplied, just send with an auth key - """ - if headers is None: - if self.auth_token is None: - self._authorize() - headers = {'Authorization': self.auth_token} - if data_file is not None: - data = data_file - else: - data = json.dumps(data) if data else None - v = version_info - headers['User-Agent'] = "duplicity version $version, " + \ - "python %s.%s.%s" % (v[0], v[1], v[2]) - - encoded_headers = dict( - (k, urllib2.quote(v.encode('utf-8'))) - for (k, v) in headers.iteritems() - ) - - try: - with OpenUrl(url, data, encoded_headers) as resp: - out = resp.read() - try: - return json.loads(out) - except ValueError: - return out - except urllib2.HTTPError as e: - self.upload_info = None - if e.code == 401: - self.auth_token = None - log.Warn("Authtoken expired, will reauthenticate with next attempt") - raise e - - def get_file_info(self, filename): - """ - Get a file info from filename - """ - endpoint = 'b2_list_file_names' - url = self.formatted_url(endpoint) - filename = self.full_filename(filename) - params = { - 'bucketId': self.bucket_id, - 'maxFileCount': 1, - 'startFileName': filename, - } - resp = self.get_or_post(url, params) - - try: - return resp['files'][0] - except IndexError: - return None - except TypeError: - return None - - def get_file_id(self, filename): - """ - Get a file id form filename - """ - try: - return self.get_file_info(filename)['fileId'] - except IndexError: - return None - except TypeError: - return None - - def full_filename(self, filename): - if self.path: - return self.path + '/' + filename - else: - return filename - - @staticmethod - def hex_sha1_of_file(path): - """ - Calculate the sha1 of a file to upload - """ - f = path.open() - block_size = 1024 * 1024 - digest = hashlib.sha1() - while True: - data = f.read(block_size) - if len(data) == 0: - break - digest.update(data) - f.close() - return digest.hexdigest() - - -class OpenUrl(object): - """ - Context manager that handles an open urllib2.Request, and provides - the file-like object that is the response. - """ - - def __init__(self, url, data, headers): - log.Log("Getting %s" % url, 9) - self.url = url - self.data = data - self.headers = headers - self.file = None - - def __enter__(self): - request = urllib2.Request(self.url, self.data, self.headers) - self.file = urllib2.urlopen(request) - log.Log("Request of %s returned with status %s" % - (self.url, self.file.code), 9) - return self.file - - def __exit__(self, exception_type, exception, traceback): - if self.file is not None: - self.file.close() + log.Log("Query: %s" % self.path + filename, log.INFO) + file_version_info = self.file_info(self.path + filename) + return {'size': file_version_info.size + if file_version_info is not None and file_version_info.size is not None else -1} + + def file_info(self, filename): + response = self.bucket.list_file_names(filename, 1) + for entry in response['files']: + file_version_info = b2.file_version.FileVersionInfoFactory.from_api_response(entry) + if file_version_info.file_name == filename: + return file_version_info + raise BackendException('File not found') duplicity.backend.register_backend("b2", B2Backend) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/duplicity/backends/pydrivebackend.py new/duplicity-0.7.15/duplicity/backends/pydrivebackend.py --- old/duplicity-0.7.14/duplicity/backends/pydrivebackend.py 2017-08-06 18:25:09.000000000 +0200 +++ new/duplicity-0.7.15/duplicity/backends/pydrivebackend.py 2017-11-01 13:27:41.000000000 +0100 @@ -211,11 +211,7 @@ if isinstance(error, FileNotUploadedError): return log.ErrorCode.backend_not_found elif isinstance(error, ApiRequestError): - http_status = error.args[0].resp.status - if http_status == 404: - return log.ErrorCode.backend_not_found - elif http_status == 403: - return log.ErrorCode.backend_permission_denied + return log.ErrorCode.backend_permission_denied return log.ErrorCode.backend_error duplicity.backend.register_backend('pydrive', PyDriveBackend) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/duplicity/collections.py new/duplicity-0.7.15/duplicity/collections.py --- old/duplicity-0.7.14/duplicity/collections.py 2017-08-30 17:04:15.000000000 +0200 +++ new/duplicity-0.7.15/duplicity/collections.py 2017-11-10 21:21:30.000000000 +0100 @@ -229,7 +229,8 @@ """ assert self.local_manifest_path manifest_buffer = self.local_manifest_path.get_data() - log.Info(_("Processing local manifest %s (%s)") % (self.local_manifest_path.name, len(manifest_buffer))) + log.Info(_("Processing local manifest %s (%s)") % ( + self.local_manifest_path.name, len(manifest_buffer))) return manifest.Manifest().from_string(manifest_buffer) def get_remote_manifest(self): @@ -241,9 +242,10 @@ manifest_buffer = self.backend.get_data(self.remote_manifest_name) except GPGError as message: log.Error(_("Error processing remote manifest (%s): %s") % - (self.remote_manifest_name, str(message))) + (util.ufn(self.remote_manifest_name), util.ufn(message))) return None - log.Info(_("Processing remote manifest %s (%s)") % (self.remote_manifest_name, len(manifest_buffer))) + log.Info(_("Processing remote manifest %s (%s)") % ( + util.ufn(self.remote_manifest_name), len(manifest_buffer))) return manifest.Manifest().from_string(manifest_buffer) def get_manifest(self): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/duplicity/dup_time.py new/duplicity-0.7.15/duplicity/dup_time.py --- old/duplicity-0.7.14/duplicity/dup_time.py 2017-08-06 18:25:09.000000000 +0200 +++ new/duplicity-0.7.15/duplicity/dup_time.py 2017-09-07 14:21:55.000000000 +0200 @@ -27,12 +27,21 @@ import types import re import calendar +import sys from duplicity import globals +from duplicity import util + +# For type testing against both int and long types that works in python 2/3 +if sys.version_info < (3,): + integer_types = (int, types.LongType) +else: + integer_types = (int,) class TimeException(Exception): pass + _interval_conv_dict = {"s": 1, "m": 60, "h": 3600, "D": 86400, "W": 7 * 86400, "M": 30 * 86400, "Y": 365 * 86400} _integer_regexp = re.compile("^[0-9]+$") @@ -69,14 +78,14 @@ """Sets the current time in curtime and curtimestr""" global curtime, curtimestr t = time_in_secs or int(time.time()) - assert type(t) in (types.LongType, types.IntType) + assert type(t) in integer_types curtime, curtimestr = t, timetostring(t) def setprevtime(time_in_secs): """Sets the previous time in prevtime and prevtimestr""" global prevtime, prevtimestr - assert type(time_in_secs) in (types.LongType, types.IntType), prevtime + assert type(time_in_secs) in integer_types, prevtime prevtime, prevtimestr = time_in_secs, timetostring(time_in_secs) @@ -181,7 +190,7 @@ if seconds == 1: partlist.append("1 second") elif not partlist or seconds > 1: - if isinstance(seconds, (types.LongType, types.IntType)): + if isinstance(seconds, integer_types): partlist.append("%s seconds" % seconds) else: partlist.append("%.2f seconds" % seconds) @@ -191,7 +200,7 @@ def intstringtoseconds(interval_string): """Convert a string expressing an interval (e.g. "4D2s") to seconds""" def error(): - raise TimeException(bad_interval_string % interval_string) + raise TimeException(bad_interval_string % util.escape(interval_string)) if len(interval_string) < 2: error() @@ -274,7 +283,7 @@ return override_curtime def error(): - raise TimeException(bad_time_string % timestr) + raise TimeException(bad_time_string % util.escape(timestr)) # Test for straight integer if _integer_regexp.search(timestr): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/duplicity/globals.py new/duplicity-0.7.15/duplicity/globals.py --- old/duplicity-0.7.14/duplicity/globals.py 2017-08-31 14:25:19.000000000 +0200 +++ new/duplicity-0.7.15/duplicity/globals.py 2017-11-13 16:56:58.000000000 +0100 @@ -26,7 +26,7 @@ # The current version of duplicity -version = "0.7.14" +version = "0.7.15" # Prefix for all files (appended before type-specific prefixes) file_prefix = "" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/duplicity/gpg.py new/duplicity-0.7.15/duplicity/gpg.py --- old/duplicity-0.7.14/duplicity/gpg.py 2017-08-28 17:27:05.000000000 +0200 +++ new/duplicity-0.7.15/duplicity/gpg.py 2017-10-26 17:52:12.000000000 +0200 @@ -94,15 +94,22 @@ _version_re = re.compile(r'^gpg.*\(GnuPG(?:/MacGPG2)?\) (?P<maj>[0-9]+)\.(?P<min>[0-9]+)\.(?P<bug>[0-9]+)(-.+)?$') def get_gpg_version(self, binary): - gpg = gpginterface.GnuPG() + gnupg = gpginterface.GnuPG() if binary is not None: - gpg.call = binary - res = gpg.run(["--version"], create_fhs=["stdout"]) + gnupg.call = binary + + # user supplied options + if globals.gpg_options: + for opt in globals.gpg_options.split(): + gnupg.options.extra_args.append(opt) + + # get gpg version + res = gnupg.run(["--version"], create_fhs=["stdout"]) line = res.handles["stdout"].readline().rstrip() m = self._version_re.search(line) if m is not None: return (int(m.group("maj")), int(m.group("min")), int(m.group("bug"))) - raise GPGError("failed to determine gpg version of %s from %s" % (binary, line)) + raise GPGError("failed to determine gnupg version of %s from %s" % (binary, line)) class GPGFile: @@ -155,7 +162,7 @@ else: raise GPGError("Unsupported GNUPG version, %s" % profile.gpg_version) - # User supplied options added later, can override ours + # user supplied options if globals.gpg_options: for opt in globals.gpg_options.split(): gnupg.options.extra_args.append(opt) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/duplicity/manifest.py new/duplicity-0.7.15/duplicity/manifest.py --- old/duplicity-0.7.14/duplicity/manifest.py 2017-08-30 21:51:07.000000000 +0200 +++ new/duplicity-0.7.15/duplicity/manifest.py 2017-10-31 15:58:00.000000000 +0100 @@ -25,6 +25,7 @@ import re +from duplicity import globals from duplicity import log from duplicity import globals from duplicity import util @@ -193,24 +194,6 @@ self.hostname = get_field("hostname") self.local_dirname = get_field("localdir") - # Get file changed list - filelist_regexp = re.compile("(^|\\n)filelist\\s([0-9]+)\\n(.*?)(\\nvolume\\s|$)", re.I | re.S) - match = filelist_regexp.search(s) - filecount = 0 - if match: - filecount = int(match.group(2)) - if filecount > 0: - def parse_fileinfo(line): - fileinfo = line.strip().split() - return (fileinfo[0], ''.join(fileinfo[1:])) - - self.files_changed = list(map(parse_fileinfo, match.group(3).split('\n'))) - - if filecount != len(self.files_changed): - log.Error(_("Manifest file '%s' is corrupt: File count says %d, File list contains %d" % - (self.fh.base if self.fh else "", filecount, len(self.files_changed)))) - self.corrupt_filelist = True - highest_vol = 0 latest_vol = 0 vi_regexp = re.compile("(?:^|\\n)(volume\\s.*(?:\\n.*)*?)(?=\\nvolume\\s|$)", re.I) @@ -228,6 +211,26 @@ for i in range(latest_vol + 1, highest_vol + 1): self.del_volume_info(i) log.Info(_("Found %s volumes in manifest") % latest_vol) + + # Get file changed list - not needed if --file-changed not present + filecount = 0 + if globals.file_changed is not None: + filelist_regexp = re.compile("(^|\\n)filelist\\s([0-9]+)\\n(.*?)(\\nvolume\\s|$)", re.I | re.S) + match = filelist_regexp.search(s) + if match: + filecount = int(match.group(2)) + if filecount > 0: + def parse_fileinfo(line): + fileinfo = line.strip().split() + return (fileinfo[0], ''.join(fileinfo[1:])) + + self.files_changed = list(map(parse_fileinfo, match.group(3).split('\n'))) + + if filecount != len(self.files_changed): + log.Error(_("Manifest file '%s' is corrupt: File count says %d, File list contains %d" % + (self.fh.base if self.fh else "", filecount, len(self.files_changed)))) + self.corrupt_filelist = True + return self def get_files_changed(self): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/duplicity/util.py new/duplicity-0.7.15/duplicity/util.py --- old/duplicity-0.7.14/duplicity/util.py 2017-05-10 22:33:23.000000000 +0200 +++ new/duplicity-0.7.15/duplicity/util.py 2017-09-07 14:19:18.000000000 +0200 @@ -48,7 +48,7 @@ msg = "Traceback (innermost last):\n" msg = msg + "%-20s %s" % (string.join(lines[:-1], ""), lines[-1]) - return uexc(msg) + return msg.decode('unicode-escape', 'replace') def escape(string): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/setup.py new/duplicity-0.7.15/setup.py --- old/duplicity-0.7.14/setup.py 2017-08-31 14:25:19.000000000 +0200 +++ new/duplicity-0.7.15/setup.py 2017-11-13 16:56:58.000000000 +0100 @@ -28,7 +28,7 @@ from setuptools.command.sdist import sdist from distutils.command.build_scripts import build_scripts -version_string = "0.7.14" +version_string = "0.7.15" if sys.version_info[:2] < (2, 6) or sys.version_info[:2] > (2, 7): print("Sorry, duplicity requires version 2.6 or 2.7 of python.") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/duplicity-0.7.14/testing/unit/test_manifest.py new/duplicity-0.7.15/testing/unit/test_manifest.py --- old/duplicity-0.7.14/testing/unit/test_manifest.py 2017-08-31 12:21:03.000000000 +0200 +++ new/duplicity-0.7.15/testing/unit/test_manifest.py 2017-10-31 17:39:52.000000000 +0100 @@ -25,6 +25,7 @@ import types import unittest +from duplicity import globals from duplicity import manifest from duplicity import path @@ -80,6 +81,15 @@ class ManifestTest(UnitTestCase): """Test Manifest class""" + + def setUp(self): + UnitTestCase.setUp(self) + self.old_files_changed = globals.file_changed + globals.file_changed = 'testing' + + def tearDown(self): + globals.file_changed = self.old_files_changed + def test_basic(self): vi1 = manifest.VolumeInfo() vi1.set_info(3, ("hello",), None, (), None)
