At file:///home/vila/src/bzr/experimental/leaking-tests/

------------------------------------------------------------
revno: 5271
revision-id: v.ladeuil...@free.fr-20100623081720-sfej05j25aopdpvy
parent: v.ladeuil...@free.fr-20100620111905-nrrk5qtd1i8lksza
committer: Vincent Ladeuil <v.ladeuil...@free.fr>
branch nick: sftp-leaks
timestamp: Wed 2010-06-23 10:17:20 +0200
message:
  Replace SocketListener by TestingTCPServerInAThread and fallouts,
  only 1 test still failing, relevant tests run now in 15s instead
  of 45s.
  
  * bzrlib/tests/stub_sftp.py:
  (StubServer): It's the test case server we're accessing here not
  the test case itself.
  (TestingSFTPConnectionHandler)
  (TestingSFTPWithoutSSHConnectionHandler, TestingSFTPServer): New
  helpers classes to split the previous implementation.
  (SFTPServer): Reimplemnted based on TestingTCPServerInAThread.
=== modified file 'bzrlib/tests/stub_sftp.py'
--- a/bzrlib/tests/stub_sftp.py 2010-06-17 09:23:27 +0000
+++ b/bzrlib/tests/stub_sftp.py 2010-06-23 08:17:20 +0000
@@ -23,6 +23,7 @@
 import paramiko
 import select
 import socket
+import SocketServer
 import sys
 import threading
 import time
@@ -38,24 +39,24 @@
 from bzrlib.tests import test_server
 
 
-class StubServer (paramiko.ServerInterface):
+class StubServer(paramiko.ServerInterface):
 
-    def __init__(self, test_case):
+    def __init__(self, test_case_server):
         paramiko.ServerInterface.__init__(self)
-        self._test_case = test_case
+        self.log = test_case_server.log
 
     def check_auth_password(self, username, password):
         # all are allowed
-        self._test_case.log('sftpserver - authorizing: %s' % (username,))
+        self.log('sftpserver - authorizing: %s' % (username,))
         return paramiko.AUTH_SUCCESSFUL
 
     def check_channel_request(self, kind, chanid):
-        self._test_case.log(
-            'sftpserver - channel request: %s, %s' % (kind, chanid))
+        self.log('sftpserver - channel request: %s, %s' % (kind, chanid))
         return paramiko.OPEN_SUCCEEDED
 
 
-class StubSFTPHandle (paramiko.SFTPHandle):
+class StubSFTPHandle(paramiko.SFTPHandle):
+
     def stat(self):
         try:
             return paramiko.SFTPAttributes.from_stat(
@@ -73,7 +74,7 @@
             return paramiko.SFTPServer.convert_errno(e.errno)
 
 
-class StubSFTPServer (paramiko.SFTPServerInterface):
+class StubSFTPServer(paramiko.SFTPServerInterface):
 
     def __init__(self, server, root, home=None):
         paramiko.SFTPServerInterface.__init__(self, server)
@@ -90,7 +91,7 @@
             self.home = home[len(self.root):]
         if self.home.startswith('/'):
             self.home = self.home[1:]
-        server._test_case.log('sftpserver - new connection')
+        server.log('sftpserver - new connection')
 
     def _realpath(self, path):
         # paths returned from self.canonicalize() always start with
@@ -241,6 +242,7 @@
     # removed: chattr, symlink, readlink
     # (nothing in bzr's sftp transport uses those)
 
+
 # ------------- server test implementation --------------
 
 STUB_SERVER_KEY = """
@@ -262,56 +264,6 @@
 """
 
 
-class SocketListener(threading.Thread):
-
-    def __init__(self, connection_callback):
-        threading.Thread.__init__(self)
-        self._connection_callback = connection_callback
-        self._socket = socket.socket()
-        self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        self._socket.bind(('localhost', 0))
-        self._socket.listen(1)
-        self.host, self.port = self._socket.getsockname()[:2]
-        self._stop_event = threading.Event()
-
-    def stop(self):
-        # called from outside this thread
-        self._stop_event.set()
-        # use a timeout here, because if the test fails, the server thread may
-        # never notice the stop_event.
-        self.join(5.0)
-        self._socket.close()
-
-    def run(self):
-        trace.mutter('SocketListener %r has started', self)
-        while True:
-            readable, writable_unused, exception_unused = \
-                select.select([self._socket], [], [], 0.1)
-            if self._stop_event.isSet():
-                trace.mutter('SocketListener %r has stopped', self)
-                return
-            if len(readable) == 0:
-                continue
-            try:
-                s, addr_unused = self._socket.accept()
-                trace.mutter('SocketListener %r has accepted connection %r',
-                    self, s)
-                # because the loopback socket is inline, and transports are
-                # never explicitly closed, best to launch a new thread.
-                threading.Thread(target=self._connection_callback,
-                                 args=(s,)).start()
-            except socket.error, x:
-                sys.excepthook(*sys.exc_info())
-                trace.warning('Socket error during accept() '
-                              'within unit test server thread: %r' % x)
-            except Exception, x:
-                # probably a failed test; unit test thread will log the
-                # failure/error
-                sys.excepthook(*sys.exc_info())
-                trace.warning(
-                    'Exception from within unit test server thread: %r' % x)
-
-
 class SocketDelay(object):
     """A socket decorator to make TCP appear slower.
 
@@ -387,58 +339,119 @@
         return bytes_sent
 
 
-class SFTPServer(test_server.TestServer):
+class TestingSFTPConnectionHandler(SocketServer.BaseRequestHandler):
+
+    def setup(self):
+        self.wrap_for_latency()
+        tcs = self.server.test_case_server
+        ssh_server = paramiko.Transport(self.request)
+        # FIXME: The server key file should be created only once, not for each
+        # connection -- vila 20100623
+        key_file = osutils.pathjoin(tcs._homedir, 'test_rsa.key')
+        f = open(key_file, 'w')
+        f.write(STUB_SERVER_KEY)
+        f.close()
+        host_key = paramiko.RSAKey.from_private_key_file(key_file)
+        ssh_server.add_server_key(host_key)
+        ssh_server.set_subsystem_handler('sftp', paramiko.SFTPServer,
+                                         StubSFTPServer, root=tcs._root,
+                                         home=tcs._server_homedir)
+        server = tcs._server_interface(tcs)
+        ssh_server.start_server(None, server)
+
+    def wrap_for_latency(self):
+        tcs = self.server.test_case_server
+        if tcs.add_latency:
+            # Give the socket (which the request really is) a latency adding
+            # decorator.
+            self.request = SocketDelay(self.request, tcs.add_latency)
+
+
+class TestingSFTPWithoutSSHConnectionHandler(TestingSFTPConnectionHandler):
+
+    def setup(self):
+        self.wrap_for_latency()
+        # Re-import these as locals, so that they're still accessible during
+        # interpreter shutdown (when all module globals get set to None, 
leading
+        # to confusing errors like "'NoneType' object has no attribute 
'error'".
+        class FakeChannel(object):
+            def get_transport(self):
+                return self
+            def get_log_channel(self):
+                return 'paramiko'
+            def get_name(self):
+                return '1'
+            def get_hexdump(self):
+                return False
+            def close(self):
+                pass
+
+        tcs = self.server.test_case_server
+        server = paramiko.SFTPServer(
+            FakeChannel(), 'sftp', StubServer(tcs), StubSFTPServer,
+            root=tcs._root, home=tcs._server_homedir)
+        try:
+            server.start_subsystem(
+                'sftp', None, ssh.SocketAsChannelAdapter(self.request))
+        except socket.error, e:
+            if (len(e.args) > 0) and (e.args[0] == errno.EPIPE):
+                # it's okay for the client to disconnect abruptly
+                # (bug in paramiko 1.6: it should absorb this exception)
+                pass
+            else:
+                raise
+        except Exception, e:
+            # This typically seems to happen during interpreter shutdown, so
+            # most of the useful ways to report this error won't work.
+            # Writing the exception type, and then the text of the exception,
+            # seems to be the best we can do.
+            # FIXME: All interpreter shutdown errors should have been related
+            # to daemon threads, cleanup needed -- vila 20100623
+            import sys
+            sys.stderr.write('\nEXCEPTION %r: ' % (e.__class__,))
+            sys.stderr.write('%s\n\n' % (e,))
+        server.finish_subsystem()
+
+
+class TestingSFTPServer(test_server.TestingThreadingTCPServer):
+
+    def __init__(self, server_address, request_handler_class, 
test_case_server):
+        test_server.TestingThreadingTCPServer.__init__(
+            self, server_address, request_handler_class)
+        self.test_case_server = test_case_server
+
+
+class SFTPServer(test_server.TestingTCPServerInAThread):
     """Common code for SFTP server facilities."""
 
     def __init__(self, server_interface=StubServer):
+        self.host = '127.0.0.1'
+        self.port = 0
+        super(SFTPServer, self).__init__((self.host, self.port),
+                                         TestingSFTPServer,
+                                         TestingSFTPConnectionHandler)
         self._original_vendor = None
-        self._homedir = None
-        self._server_homedir = None
-        self._listener = None
-        self._root = None
         self._vendor = ssh.ParamikoVendor()
         self._server_interface = server_interface
-        # sftp server logs
         self.logs = []
         self.add_latency = 0
+        self._homedir = None
+        self._server_homedir = None
+        self._root = None
 
     def _get_sftp_url(self, path):
         """Calculate an sftp url to this server for path."""
-        return 'sftp://foo:b...@%s:%d/%s' % (self._listener.host,
-                                            self._listener.port, path)
+        return "sftp://foo:b...@%s:%s/%s"; % (self.host, self.port, path)
 
     def log(self, message):
         """StubServer uses this to log when a new server is created."""
         self.logs.append(message)
 
-    def _run_server_entry(self, sock):
-        """Entry point for all implementations of _run_server.
-
-        If self.add_latency is > 0.000001 then sock is given a latency adding
-        decorator.
-        """
-        if self.add_latency > 0.000001:
-            # FIXME: We appear to use SocketDelay for the server socket only, I
-            # don't think that gets magically propagated to client
-            # sockets... -- vila 20100526
-            sock = SocketDelay(sock, self.add_latency)
-        return self._run_server(sock)
-
-    def _run_server(self, s):
-        ssh_server = paramiko.Transport(s)
-        key_file = osutils.pathjoin(self._homedir, 'test_rsa.key')
-        f = open(key_file, 'w')
-        f.write(STUB_SERVER_KEY)
-        f.close()
-        host_key = paramiko.RSAKey.from_private_key_file(key_file)
-        ssh_server.add_server_key(host_key)
-        server = self._server_interface(self)
-        ssh_server.set_subsystem_handler('sftp', paramiko.SFTPServer,
-                                         StubSFTPServer, root=self._root,
-                                         home=self._server_homedir)
-        ready = threading.Event()
-        ssh_server.start_server(ready, server)
-        ready.wait(5.0)
+    def create_server(self):
+        server = self.server_class((self.host, self.port),
+                                   self.request_handler_class,
+                                   self)
+        return server
 
     def start_server(self, backing_server=None):
         # XXX: TODO: make sftpserver back onto backing_server rather than local
@@ -463,17 +476,17 @@
         self._root = '/'
         if sys.platform == 'win32':
             self._root = ''
-        self._listener = SocketListener(self._run_server_entry)
-        self._listener.setDaemon(True)
-        self._listener.start()
+        super(SFTPServer, self).start_server()
 
     def stop_server(self):
-        self._listener.stop()
-        ssh._ssh_vendor_manager._cached_ssh_vendor = self._original_vendor
+        try:
+            super(SFTPServer, self).stop_server()
+        finally:
+            ssh._ssh_vendor_manager._cached_ssh_vendor = self._original_vendor
 
     def get_bogus_url(self):
         """See bzrlib.transport.Server.get_bogus_url."""
-        # this is chosen to try to prevent trouble with proxies, wierd dns, etc
+        # this is chosen to try to prevent trouble with proxies, weird dns, etc
         # we bind a random socket, so that we get a guaranteed unused port
         # we just never listen on that port
         s = socket.socket()
@@ -499,45 +512,7 @@
     def __init__(self):
         super(SFTPServerWithoutSSH, self).__init__()
         self._vendor = ssh.LoopbackVendor()
-
-    def _run_server(self, sock):
-        # Re-import these as locals, so that they're still accessible during
-        # interpreter shutdown (when all module globals get set to None, 
leading
-        # to confusing errors like "'NoneType' object has no attribute 
'error'".
-        class FakeChannel(object):
-            def get_transport(self):
-                return self
-            def get_log_channel(self):
-                return 'paramiko'
-            def get_name(self):
-                return '1'
-            def get_hexdump(self):
-                return False
-            def close(self):
-                pass
-
-        server = paramiko.SFTPServer(
-            FakeChannel(), 'sftp', StubServer(self), StubSFTPServer,
-            root=self._root, home=self._server_homedir)
-        try:
-            server.start_subsystem(
-                'sftp', None, ssh.SocketAsChannelAdapter(sock))
-        except socket.error, e:
-            if (len(e.args) > 0) and (e.args[0] == errno.EPIPE):
-                # it's okay for the client to disconnect abruptly
-                # (bug in paramiko 1.6: it should absorb this exception)
-                pass
-            else:
-                raise
-        except Exception, e:
-            # This typically seems to happen during interpreter shutdown, so
-            # most of the useful ways to report this error won't work.
-            # Writing the exception type, and then the text of the exception,
-            # seems to be the best we can do.
-            import sys
-            sys.stderr.write('\nEXCEPTION %r: ' % (e.__class__,))
-            sys.stderr.write('%s\n\n' % (e,))
-        server.finish_subsystem()
+        self.request_handler_class = TestingSFTPWithoutSSHConnectionHandler
 
 
 class SFTPAbsoluteServer(SFTPServerWithoutSSH):
@@ -566,7 +541,9 @@
     It does this by serving from a deeply-nested directory that doesn't exist.
     """
 
-    def start_server(self, backing_server=None):
-        self._server_homedir = '/dev/noone/runs/tests/here'
-        super(SFTPSiblingAbsoluteServer, self).start_server(backing_server)
+    def create_server(self):
+        # FIXME: Can't we do that in a cleaner way ? -- vila 20100623
+        server = super(SFTPSiblingAbsoluteServer, self).create_server()
+        server._server_homedir = '/dev/noone/runs/tests/here'
+        return server
 

=== modified file 'bzrlib/tests/test_sftp_transport.py'
--- a/bzrlib/tests/test_sftp_transport.py       2010-06-20 11:18:38 +0000
+++ b/bzrlib/tests/test_sftp_transport.py       2010-06-23 08:17:20 +0000
@@ -504,7 +504,7 @@
     """Test that AuthenticationConfig can supply default usernames."""
 
     def get_transport_for_connection(self, set_config):
-        port = self.get_server()._listener.port
+        port = self.get_server().port
         if set_config:
             conf = config.AuthenticationConfig()
             conf._get_config().update(

-- 
bazaar-commits mailing list
bazaar-commits@lists.canonical.com
https://lists.ubuntu.com/mailman/listinfo/bazaar-commits

Reply via email to