edso has proposed merging lp:~ed.so/duplicity/webdav.fix-retry into lp:duplicity.
Requested reviews: duplicity-team (duplicity-team) For more details, see: https://code.launchpad.net/~ed.so/duplicity/webdav.fix-retry/+merge/142759 webdav - added ssl certificate verification (see man page) - more robust retry routine to survive ssl errors, broken pipe errors - added http redirect support -- https://code.launchpad.net/~ed.so/duplicity/webdav.fix-retry/+merge/142759 Your team duplicity-team is requested to review the proposed merge of lp:~ed.so/duplicity/webdav.fix-retry into lp:duplicity.
=== modified file 'bin/duplicity.1' --- bin/duplicity.1 2012-12-25 11:07:32 +0000 +++ bin/duplicity.1 2013-01-10 19:14:20 +0000 @@ -64,6 +64,7 @@ .B ssh pexpect backend .B sftp/scp client binaries OpenSSH - http://www.openssh.com/ +<<<<<<< TREE .TP .B "Ubuntu One" .B httplib2 @@ -73,6 +74,13 @@ .B oauthlib (python OAuth request-signing logic) - http://pypi.python.org/pypi/oauthlib +======= +.TP +.B "webdav backend" +.B ca cert file +for ssl certificate verification of HTTPS connections +- http://curl.haxx.se/docs/caextract.html +>>>>>>> MERGE-SOURCE .SH SYNOPSIS .B duplicity [full|incremental] @@ -773,6 +781,22 @@ .RE .TP +.BI "--ssl-cacert-file " file +.B (only webdav backend) +Provide a cacert file for ssl certificate verification. +.br +See also +.BR "A NOTE ON SSL CERTIFICATE VERIFICATION" . + +.TP +.B --ssl-no-check-certificate +.B (only webdav backend) +Disable ssl certificate verification. +.br +See also +.BR "A NOTE ON SSL CERTIFICATE VERIFICATION" . + +.TP .BI "--tempdir " directory Use this existing directory for duplicity temporary files instead of the system default, which is usually the /tmp directory. This option @@ -1375,6 +1399,7 @@ in addition for this backend mode to work properly. Sftp does not have these limitations but needs an sftp service running on the backend server, which is sometimes not an option. +<<<<<<< TREE .SH A NOTE ON UBUNTU ONE To use Ubuntu One you must have an Ubuntu One OAuth access token. Such @@ -1389,6 +1414,40 @@ acquired token is then printed to the console. .PP See https://one.ubuntu.com/ for more information about Ubuntu One. +======= +.SH A NOTE ON SSL CERTIFICATE VERIFICATION +Certificate verification as implemented right now [01.2013] only in the webdav backend needs a file +based database of certification authority certificates (cacert file). It has to be a +.B PEM +formatted text file as currently provided by the +.B CURL +project. See +.PP +.RS +http://curl.haxx.se/docs/caextract.html +.PP +.RE +After creating/retrieving a valid cacert file you should copy it to either +.PP +.RS +~/.duplicity/cacert.pem +.br +~/duplicity_cacert.pem +.br +/etc/duplicity/cacert.pem +.PP +.RE +Duplicity searches it there in the same order and will fail if it can't find it. +You can however specify the option +.BI --ssl-cacert-file " <file>" +to point duplicity to a copy in a different location. +.PP +Finally there is the +.B --ssl-no-check-certificate +option to disable certificate verification alltogether, in case some ssl library +is missing or verification is not wanted. Use it with care, as even with self signed +servers manually providing the private ca certificate is definitely the safer option. +>>>>>>> MERGE-SOURCE .SH A NOTE ON SYMMETRIC ENCRYPTION AND SIGNING Signing and symmetrically encrypt at the same time with the gpg binary on the @@ -1409,6 +1468,20 @@ .BI PASSPHRASE for symmetric encryption and the passphrase of the signing key are identical. +.SH A NOTE ON UBUNTU ONE + +To use Ubuntu One you must have an Ubuntu One OAuth access token. Such +OAuth tokens have a practically unlimited lifetime; you can have multiple +active tokens and you can revoke tokens using the Ubuntu One web interface. +.PP +duplicity expects th token in the environment variable FTP_PASSWORD +(in the format "consumer_key:consumer_secret:token:token_secret"). If no +token is present, duplicity asks for your Ubuntu One email address and password +and requests an access token from the Ubuntu SSO service. The newly +acquired token is then printed to the console. +.PP +See https://one.ubuntu.com/ for more information about Ubuntu One. + .SH KNOWN ISSUES / BUGS Hard links currently unsupported (they will be treated as non-linked regular files). === modified file 'duplicity/backend.py' --- duplicity/backend.py 2012-12-12 17:45:38 +0000 +++ duplicity/backend.py 2013-01-10 19:14:20 +0000 @@ -42,7 +42,7 @@ from duplicity.util import exception_traceback -from duplicity.errors import BackendException +from duplicity.errors import BackendException, FatalBackendError from duplicity.errors import TemporaryLoadException from duplicity.errors import ConflictingScheme from duplicity.errors import InvalidBackendURL @@ -333,26 +333,35 @@ # as we don't know what the underlying code comes up with and we really *do* # want to retry globals.num_retries times under all circumstances def retry_fatal(fn): - def iterate(*args): - for n in range(1, globals.num_retries): - try: - return fn(*args) - except Exception, e: - log.Warn("Attempt %s failed. %s: %s" - % (n, e.__class__.__name__, str(e))) - log.Debug("Backtrace of previous error: %s" - % exception_traceback()) - time.sleep(10) # wait a bit before trying again + def _retry_fatal(self, *args): + try: + n = 0 + for n in range(1, globals.num_retries): + try: + self.retry_count = n + return fn(self, *args) + except FatalBackendError, e: + # die on fatal errors + raise e + except Exception, e: + # retry on anything else + log.Warn("Attempt %s failed. %s: %s" + % (n, e.__class__.__name__, str(e))) + log.Debug("Backtrace of previous error: %s" + % exception_traceback()) + time.sleep(10) # wait a bit before trying again # final trial, die on exception - try: - return fn(*args) + self.retry_count = n+1 + return fn(self, *args) except Exception, e: log.FatalError("Giving up after %s attempts. %s: %s" - % (globals.num_retries, e.__class__.__name__, str(e)), + % (self.retry_count, e.__class__.__name__, str(e)), log.ErrorCode.backend_error) log.Debug("Backtrace of previous error: %s" % exception_traceback()) - return iterate + self.retry_count = 0 + + return _retry_fatal class Backend: """ @@ -371,6 +380,7 @@ - move """ + def __init__(self, parsed_url): self.parsed_url = parsed_url === modified file 'duplicity/backends/webdavbackend.py' --- duplicity/backends/webdavbackend.py 2012-12-09 14:36:19 +0000 +++ duplicity/backends/webdavbackend.py 2013-01-10 19:14:20 +0000 @@ -2,6 +2,8 @@ # # Copyright 2002 Ben Escoto <[email protected]> # Copyright 2007 Kenneth Loafman <[email protected]> +# Copyright 2013 Edgar Soldin +# - ssl cert verification, some robustness enhancements # # This file is part of duplicity. # @@ -20,7 +22,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import base64 -import httplib +import httplib, os import re import urllib import urllib2 @@ -46,6 +48,58 @@ def get_method(self): return self.method +class VerifiedHTTPSConnection(httplib.HTTPSConnection): + def __init__(self, *args, **kwargs): + try: + global socket, ssl + import socket, ssl + except ImportError: + raise FatalBackendError("Missing socket or ssl libraries.") + + httplib.HTTPSConnection.__init__(self, *args, **kwargs) + + self.cacert_file = globals.ssl_cacert_file + cacert_candidates = [ "~/.duplicity/cacert.pem", \ + "~/duplicity_cacert.pem", \ + "/etc/duplicity/cacert.pem" ] + # + if not self.cacert_file: + for path in cacert_candidates : + path = os.path.expanduser(path) + if (os.path.isfile(path)): + self.cacert_file = path + break + # still no cacert file, inform user + if not self.cacert_file: + raise FatalBackendError("""For certificate verification a cacert database file is needed in one of these locations: %s +Hints: + Consult the man page, chapter 'SSL Certificate Verification'. + Consider using the options --ssl-cacert-file, --ssl-no-check-certificate .""" % ", ".join(cacert_candidates) ) + # check if file is accessible (libssl errors are not very detailed) + if not os.access(self.cacert_file, os.R_OK): + raise FatalBackendError("Cacert database file '%s' is not readable." % cacert_file) + + def connect(self): + # create new socket + sock = socket.create_connection((self.host, self.port), + self.timeout) + if self._tunnel_host: + self.sock = sock + self._tunnel() + + # wrap the socket in ssl using verification + self.sock = ssl.wrap_socket(sock, + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=self.cacert_file, + ) + + def request(self, *args, **kwargs): + try: + return httplib.HTTPSConnection.request(self, *args, **kwargs) + except ssl.SSLError, e: + # encapsulate ssl errors + raise BackendException("SSL failed: %s" % str(e),log.ErrorCode.backend_error) + class WebDAVBackend(duplicity.backend.Backend): """Backend for accessing a WebDAV repository. @@ -70,23 +124,22 @@ self.digest_challenge = None self.digest_auth_handler = None - if parsed_url.path: + self.username = parsed_url.username + self.password = self.get_password() + self.directory = self._sanitize_path(parsed_url.path) + + log.Info("Using WebDAV protocol %s" % (globals.webdav_proto,)) + log.Info("Using WebDAV host %s port %s" % (parsed_url.hostname, parsed_url.port)) + log.Info("Using WebDAV directory %s" % (self.directory,)) + + self.conn = None + + def _sanitize_path(self,path): + if path: foldpath = re.compile('/+') - self.directory = foldpath.sub('/', parsed_url.path + '/' ) - else: - self.directory = '/' - - log.Info("Using WebDAV host %s" % (parsed_url.hostname,)) - log.Info("Using WebDAV port %s" % (parsed_url.port,)) - log.Info("Using WebDAV directory %s" % (self.directory,)) - log.Info("Using WebDAV protocol %s" % (globals.webdav_proto,)) - - if parsed_url.scheme == 'webdav': - self.conn = httplib.HTTPConnection(parsed_url.hostname, parsed_url.port) - elif parsed_url.scheme == 'webdavs': - self.conn = httplib.HTTPSConnection(parsed_url.hostname, parsed_url.port) - else: - raise BackendException("Unknown URI scheme: %s" % (parsed_url.scheme)) + return foldpath.sub('/', path + '/' ) + else: + return '/' def _getText(self,nodelist): rc = "" @@ -94,29 +147,76 @@ if node.nodeType == node.TEXT_NODE: rc = rc + node.data return rc + + def _connect(self, forced=False): + """ + Connect or re-connect to the server, updates self.conn + # reconnect on errors as a precaution, there are errors e.g. + # "[Errno 32] Broken pipe" or SSl errors that render the connection unusable + """ + if self.retry_count<=1 and self.conn \ + and self.conn.host == self.parsed_url.hostname: return + + log.Info("WebDAV create connection on '%s' (retry %s) " % (self.parsed_url.hostname,self.retry_count) ) + if self.conn: self.conn.close() + # http schemes needed for redirect urls from servers + if self.parsed_url.scheme in ['webdav','http']: + self.conn = httplib.HTTPConnection(self.parsed_url.hostname, self.parsed_url.port) + elif self.parsed_url.scheme in ['webdavs','https']: + if globals.ssl_no_check_certificate: + self.conn = httplib.HTTPSConnection(self.parsed_url.hostname, self.parsed_url.port) + else: + self.conn = VerifiedHTTPSConnection(self.parsed_url.hostname, self.parsed_url.port) + else: + raise FatalBackendError("WebDAV Unknown URI scheme: %s" % (self.parsed_url.scheme)) def close(self): self.conn.close() - def request(self, method, path, data=None): + def request(self, method, path, data=None, redirected=0): """ Wraps the connection.request method to retry once if authentication is required """ - quoted_path = urllib.quote(path) + self._connect() + + quoted_path = urllib.quote(path,"/:~") if self.digest_challenge is not None: self.headers['Authorization'] = self.get_digest_authorization(path) + + log.Info("WebDAV %s %s request with headers: %s " % (method,quoted_path,self.headers)) + log.Info("WebDAV data length: %s " % len(str(data)) ) self.conn.request(method, quoted_path, data, self.headers) response = self.conn.getresponse() - if response.status == 401: + log.Info("WebDAV response status %s with reason '%s'." % (response.status,response.reason)) + # resolve redirects and reset url on listing requests (they usually come before everything else) + if response.status in [301,302] and method == 'PROPFIND': + redirect_url = response.getheader('location',None) + response.close() + if redirect_url: + log.Notice("WebDAV redirect to: %s " % urllib.unquote(redirect_url) ) + if redirected > 10: + raise FatalBackendError("WebDAV redirected 10 times. Giving up.") + self.parsed_url = duplicity.backend.ParsedUrl(redirect_url) + self.directory = self._sanitize_path(self.parsed_url.path) + return self.request(method,self.directory,data,redirected+1) + else: + raise FatalBackendError("WebDAV missing location header in redirect response.") + elif response.status == 401: response.close() self.headers['Authorization'] = self.get_authorization(response, quoted_path) + log.Info("WebDAV retry request with authentification headers.") + log.Info("WebDAV %s %s request2 with headers: %s " % (method,quoted_path,self.headers)) + log.Info("WebDAV data length: %s " % len(str(data)) ) self.conn.request(method, quoted_path, data, self.headers) response = self.conn.getresponse() - + log.Info("WebDAV response2 status %s with reason '%s'." % (response.status,response.reason)) + return response + + def get_authorization(self, response, path): """ Fetches the auth header based on the requested method (basic or digest) @@ -139,7 +239,7 @@ """ Returns the basic auth header """ - auth_string = '%s:%s' % (self.parsed_url.username, self.get_password()) + auth_string = '%s:%s' % (self.username, self.password) return 'Basic %s' % base64.encodestring(auth_string).strip() def get_digest_authorization(self, path): @@ -149,7 +249,7 @@ u = self.parsed_url if self.digest_auth_handler is None: pw_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() - pw_manager.add_password(None, self.conn.host, u.username, self.get_password()) + pw_manager.add_password(None, self.conn.host, self.username, self.password) self.digest_auth_handler = urllib2.HTTPDigestAuthHandler(pw_manager) # building a dummy request that gets never sent, @@ -165,6 +265,7 @@ def list(self): """List files in directory""" log.Info("Listing directory %s on WebDAV server" % (self.directory,)) + response = None try: self.headers['Depth'] = "1" response = self.request("PROPFIND", self.directory, self.listbody) @@ -178,7 +279,7 @@ response.close() # just created folder is so return empty return [] - elif response.status == 207: + elif response.status in [200, 207]: document = response.read() response.close() else: @@ -195,9 +296,10 @@ if filename: result.append(filename) return result - except Exception, cause: - e = BackendException("Listing directory %s on WebDAV server failed. %s" % (self.directory,cause)) + except Exception, e: raise e + finally: + if response: response.close() def __taste_href(self, href): """ @@ -241,11 +343,15 @@ """Get remote filename, saving it to local_path""" url = self.directory + remote_filename log.Info("Retrieving %s from WebDAV server" % (url ,)) + response = None try: target_file = local_path.open("wb") response = self.request("GET", url) if response.status == 200: + #data=response.read() target_file.write(response.read()) + #import hashlib + #log.Info("WebDAV GOT %s bytes with md5=%s" % (len(data),hashlib.md5(data).hexdigest()) ) assert not target_file.close() local_path.setdata() response.close() @@ -254,9 +360,10 @@ reason = response.reason response.close() raise BackendException("Bad status code %s reason %s." % (status,reason)) - except Exception, cause: - e = BackendException("Getting %s from WebDAV server failed. %s" % (url,cause)) + except Exception, e: raise e + finally: + if response: response.close() @retry_fatal def put(self, source_path, remote_filename = None): @@ -265,10 +372,11 @@ remote_filename = source_path.get_filename() url = self.directory + remote_filename log.Info("Saving %s on WebDAV server" % (url ,)) + response = None try: source_file = source_path.open("rb") response = self.request("PUT", url, source_file.read()) - if response.status == 201: + if response.status in [201, 204]: response.read() response.close() else: @@ -276,9 +384,10 @@ reason = response.reason response.close() raise BackendException("Bad status code %s reason %s." % (status,reason)) - except Exception, cause: - e = BackendException("Putting %s on WebDAV server failed. %s" % (url,cause)) + except Exception, e: raise e + finally: + if response: response.close() @retry_fatal def delete(self, filename_list): @@ -286,6 +395,7 @@ for filename in filename_list: url = self.directory + filename log.Info("Deleting %s from WebDAV server" % (url ,)) + response = None try: response = self.request("DELETE", url) if response.status == 204: @@ -296,9 +406,10 @@ reason = response.reason response.close() raise BackendException("Bad status code %s reason %s." % (status,reason)) - except Exception, cause: - e = BackendException("Deleting %s on WebDAV server failed. %s" % (url,cause)) + except Exception, e: raise e + finally: + if response: response.close() duplicity.backend.register_backend("webdav", WebDAVBackend) duplicity.backend.register_backend("webdavs", WebDAVBackend) === modified file 'duplicity/commandline.py' --- duplicity/commandline.py 2012-11-21 01:27:35 +0000 +++ duplicity/commandline.py 2013-01-10 19:14:20 +0000 @@ -502,6 +502,11 @@ # user added ssh options parser.add_option("--ssh-options", action="extend", metavar=_("options")) + # user added ssl options (webdav backend) + parser.add_option("--ssl-cacert-file", metavar=_("pem formatted bundle of certificate authorities")) + + parser.add_option("--ssl-no-check-certificate", action="store_true") + # Working directory for the tempfile module. Defaults to /tmp on most systems. parser.add_option("--tempdir", dest="temproot", type="file", metavar=_("path")) === modified file 'duplicity/errors.py' --- duplicity/errors.py 2011-08-03 19:47:29 +0000 +++ duplicity/errors.py 2013-01-10 19:14:20 +0000 @@ -70,6 +70,12 @@ """ pass +class FatalBackendError(DuplicityError): + """ + Raised to indicate a backend failed fatally. + """ + pass + class TemporaryLoadException(BackendException): """ Raised to indicate a temporary issue on the backend. === modified file 'duplicity/globals.py' --- duplicity/globals.py 2012-03-13 20:53:35 +0000 +++ duplicity/globals.py 2013-01-10 19:14:20 +0000 @@ -214,6 +214,10 @@ # whether to use scp for put/get, sftp is default use_scp = False +# HTTPS ssl optons (currently only webdav) +ssl_cacert_file = None +ssl_no_check_certificate = False + # user added rsync options rsync_options = ""
_______________________________________________ Mailing list: https://launchpad.net/~duplicity-team Post to : [email protected] Unsubscribe : https://launchpad.net/~duplicity-team More help : https://help.launchpad.net/ListHelp
