Thiago F. Pappacena has proposed merging ~pappacena/launchpad:gunicorn-appserver-tests into launchpad:master.
Commit message: Run tests with gunicorn instead of zope server (disabled) Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~pappacena/launchpad/+git/launchpad/+merge/396691 -- Your team Launchpad code reviewers is requested to review the proposed merge of ~pappacena/launchpad:gunicorn-appserver-tests into launchpad:master.
diff --git a/configs/development/launchpad-lazr.conf b/configs/development/launchpad-lazr.conf index a4956f3..8409e5c 100644 --- a/configs/development/launchpad-lazr.conf +++ b/configs/development/launchpad-lazr.conf @@ -88,6 +88,7 @@ public_host: keyserver.launchpad.test public_https: False [launchpad] +devmode: true enable_test_openid_provider: True openid_canonical_root: https://testopenid.test/ openid_provider_root: https://testopenid.test/ @@ -223,6 +224,7 @@ hostname: xmlrpc.launchpad.test [vhost.xmlrpc_private] hostname: xmlrpc-private.launchpad.test +private_port: 8087 [vhost.feeds] hostname: feeds.launchpad.test diff --git a/configs/testrunner-appserver/gunicorn.conf.py b/configs/testrunner-appserver/gunicorn.conf.py new file mode 100644 index 0000000..ba22ec1 --- /dev/null +++ b/configs/testrunner-appserver/gunicorn.conf.py @@ -0,0 +1,12 @@ +import os +config_dir = os.path.dirname(__file__) +log_dir = os.path.join(config_dir, '..', '..', 'logs') + +bind = [":8085", ":8087"] +workers = 1 +threads = 10 +log_level = "DEBUG" + +log_file = os.path.join(log_dir, 'gunicorn.log') +error_logfile = os.path.join(log_dir, 'gunicorn-error.log') +access_logfile = os.path.join(log_dir, 'gunicorn-access.log') diff --git a/configs/testrunner/gunicorn.conf.py b/configs/testrunner/gunicorn.conf.py new file mode 100644 index 0000000..ba22ec1 --- /dev/null +++ b/configs/testrunner/gunicorn.conf.py @@ -0,0 +1,12 @@ +import os +config_dir = os.path.dirname(__file__) +log_dir = os.path.join(config_dir, '..', '..', 'logs') + +bind = [":8085", ":8087"] +workers = 1 +threads = 10 +log_level = "DEBUG" + +log_file = os.path.join(log_dir, 'gunicorn.log') +error_logfile = os.path.join(log_dir, 'gunicorn-error.log') +access_logfile = os.path.join(log_dir, 'gunicorn-access.log') diff --git a/lib/lp/scripts/runlaunchpad.py b/lib/lp/scripts/runlaunchpad.py index 0557ee8..79ba3d1 100644 --- a/lib/lp/scripts/runlaunchpad.py +++ b/lib/lp/scripts/runlaunchpad.py @@ -6,11 +6,13 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = ['start_launchpad'] +import tempfile try: from contextlib import ExitStack except ImportError: from contextlib2 import ExitStack import os +import re import signal import subprocess import sys @@ -332,7 +334,31 @@ def start_testapp(argv=list(sys.argv)): pass +def gunicornify_zope_config_file(): + """Creates a new launchpad.config file removing directives related to + Zope Server that shouldn't be used when running on gunicorn. + """ + original_filename = config.zope_config_file + with open(original_filename) as fd: + content = fd.read() + + # Remove unwanted tags. + for tag in ['server', 'accesslog']: + content = re.sub( + r"<%s>(.*)</%s>" % (tag, tag), "", content, flags=re.S) + + # Remove unwanted single-line directives. + for directive in ['interrupt-check-interval']: + content = re.sub(r"%s .*" % directive, "", content) + + new_filename = tempfile.mktemp(prefix='launchpad.conf-gunicorn-') + with open(new_filename, 'w') as fd: + fd.write(content) + config.zope_config_file = new_filename + + def gunicorn_main(): + gunicornify_zope_config_file() orig_argv = sys.argv try: sys.argv = [ diff --git a/lib/lp/scripts/tests/test_runlaunchpad.py b/lib/lp/scripts/tests/test_runlaunchpad.py index 4a7222c..9587583 100644 --- a/lib/lp/scripts/tests/test_runlaunchpad.py +++ b/lib/lp/scripts/tests/test_runlaunchpad.py @@ -15,11 +15,13 @@ __all__ = [ import os import shutil import tempfile +from textwrap import dedent import testtools from lp.scripts.runlaunchpad import ( get_services_to_run, + gunicornify_zope_config_file, process_config_arguments, SERVICES, split_out_runlaunchpad_arguments, @@ -191,3 +193,56 @@ class TestAppServerStart(lp.testing.TestCase): start_launchpad([]) self.assertEqual(0, gmain.call_count) self.assertEqual(1, zmain.call_count) + + def test_gunicornify_config(self): + content = dedent(""" + site-definition zcml/webapp.zcml + # With some comment + devmode off + interrupt-check-interval 200 + <server> + type HTTP + address 8085 + </server> + <server> + type XXX + address 123 + </server> + + <zodb> + <mappingstorage/> + </zodb> + + <accesslog> + <logfile> + path logs/test-appserver-layer.log + </logfile> + </accesslog> + """) + config_filename = tempfile.mktemp() + with open(config_filename, "w") as fd: + fd.write(content) + + patched_cfg = mock.patch( + 'lp.services.config.LaunchpadConfig.zope_config_file', + new_callable=mock.PropertyMock) + with patched_cfg as mock_zope_config_file: + mock_zope_config_file.return_value = config_filename + + gunicornify_zope_config_file() + self.assertEqual(2, mock_zope_config_file.call_count) + new_file = mock_zope_config_file.call_args[0][0] + with open(new_file) as fd: + self.assertEqual(dedent(""" + site-definition zcml/webapp.zcml + # With some comment + devmode off + + + + <zodb> + <mappingstorage/> + </zodb> + + + """), fd.read()) diff --git a/lib/lp/services/config/__init__.py b/lib/lp/services/config/__init__.py index d4bd705..c2ed613 100644 --- a/lib/lp/services/config/__init__.py +++ b/lib/lp/services/config/__init__.py @@ -125,6 +125,12 @@ class LaunchpadConfig: self._process_name = process_name self._instance_name = instance_name self.root = TREE_ROOT + # Allow overriding Zope's config file. + # XXX pappacena 2021-01-21: We allow overriding this at runtime so + # we can do a painless transition to gunicorn. Once we are fully + # running gunicorn instead of zope server, this shouldn't be + # necessary anymore. + self._zope_config_file = None def _make_process_name(self): if getattr(sys, 'argv', None) is None: @@ -152,7 +158,7 @@ class LaunchpadConfig: """When running launchpad server, shall we use gunicorn?""" # XXX pappacena: 2021-01-20: Forced False until we have everything # in place. - return False + return True def setInstance(self, instance_name): """Set the instance name where the conf files are stored. @@ -251,10 +257,22 @@ class LaunchpadConfig: @property def zope_config_file(self): """Return the path to the ZConfig file for this instance.""" + if self._zope_config_file is not None: + return self._zope_config_file return os.path.join(self.config_dir, 'launchpad.conf') + @zope_config_file.setter + def zope_config_file(self, value): + self._zope_config_file = value + def _getZConfig(self): - """Modify the config, adding automatically generated settings""" + """Modify the config, adding automatically generated settings + + XXX pappacena 2021-01-21 This method should probably be deprecated + once we move to gunicorn since we read zope's configs here mostly to + get "devmode" directive, which is moved to launchpad-lazr.conf when + running launchpad server on gunicorn. + """ with resources.path('zope.app.server', 'schema.xml') as schemafile: schema = ZConfig.loadSchema(str(schemafile)) root_options, handlers = ZConfig.loadConfig( @@ -268,6 +286,10 @@ class LaunchpadConfig: Copied here for ease of access. """ + if self.use_gunicorn: + return self.launchpad.devmode + # XXX pappacena 2021-01-21: Read only from launchpad-lazr.conf file + # once we will be running on gunicorn. if self._devmode is None: self._getZConfig() return self._devmode @@ -279,6 +301,8 @@ class LaunchpadConfig: @property def servers(self): """The defined servers.""" + if self.use_gunicorn: + return [] if self._servers is None: self._getZConfig() return self._servers diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf index aaa9d30..6b54929 100644 --- a/lib/lp/services/config/schema-lazr.conf +++ b/lib/lp/services/config/schema-lazr.conf @@ -843,6 +843,9 @@ max_scaling: 500 [launchpad] +# Is this server running in dev mode? +devmode: false + # A directory of files from which *-lazr.conf will be loaded and # overlaid in order on top of the main config. # Note that this is relative to the top-level config file, not @@ -1640,6 +1643,10 @@ althostnames: none # datatype: string rooturl: none +# Is this vhost enabled? +# Used to create the correct publishers +private_port: None + [vhost.mainsite] # Can the profile page act as a OpenID delegated identity? # data-type: boolean diff --git a/lib/lp/services/webapp/publication.py b/lib/lp/services/webapp/publication.py index 7325d9f..25bc2de 100644 --- a/lib/lp/services/webapp/publication.py +++ b/lib/lp/services/webapp/publication.py @@ -1,4 +1,4 @@ -# Copyright 2009-2019 Canonical Ltd. This software is licensed under the +# Copyright 2009-2021 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). __metaclass__ = type @@ -7,6 +7,7 @@ __all__ = [ 'LaunchpadBrowserPublication', ] +import logging import re import sys import threading @@ -871,6 +872,11 @@ def tracelog(request, prefix, msg): easier. ``prefix`` should be unique and contain no spaces, and preferably a single character to save space. """ - tracelog = ITraceLog(request, None) - if tracelog is not None: - tracelog.log('%s %s' % (prefix, six.ensure_str(msg, 'US-ASCII'))) + msg = '%s %s' % (prefix, six.ensure_str(msg, 'US-ASCII')) + if not config.use_gunicorn: + tracelog = ITraceLog(request, None) + if tracelog is not None: + tracelog.log(msg) + else: + logger = logging.getLogger() + logger.info(msg) diff --git a/lib/lp/services/webapp/servers.py b/lib/lp/services/webapp/servers.py index d7bcad2..0e2bf17 100644 --- a/lib/lp/services/webapp/servers.py +++ b/lib/lp/services/webapp/servers.py @@ -1,4 +1,4 @@ -# Copyright 2009-2019 Canonical Ltd. This software is licensed under the +# Copyright 2009-2021 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Definition of the internet servers that Launchpad uses.""" @@ -1110,7 +1110,8 @@ class LaunchpadAccessLogger(CommonAccessLogger): ) ) - +# XXX pappacena 2021-01-21: These 4 server definitions can be removed once +# we are using only gunicorn (and not Zope Server). http = wsgi.ServerType( ZServerTracelogServer, # subclass of WSGIHTTPServer WSGIPublisherApplication, @@ -1577,10 +1578,13 @@ def register_launchpad_request_publication_factories(): # We may also have a private XML-RPC server. private_port = None - for server in config.servers: - if server.type == 'PrivateXMLRPC': - ip, private_port = server.address - break + if config.use_gunicorn: + private_port = config.vhost.xmlrpc_private.private_port + else: + for server in config.servers: + if server.type == 'PrivateXMLRPC': + ip, private_port = server.address + break if private_port is not None: factories.append(XMLRPCRequestPublicationFactory( diff --git a/lib/lp/testing/layers.py b/lib/lp/testing/layers.py index 9089c60..98c8149 100644 --- a/lib/lp/testing/layers.py +++ b/lib/lp/testing/layers.py @@ -1,4 +1,4 @@ -# Copyright 2009-2019 Canonical Ltd. This software is licensed under the +# Copyright 2009-2021 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Layers used by Launchpad tests. @@ -55,6 +55,7 @@ from functools import partial import gc import logging import os +import select import signal import socket import subprocess @@ -74,7 +75,10 @@ from fixtures import ( MonkeyPatch, ) import psycopg2 -from six.moves.urllib.error import URLError +from six.moves.urllib.error import ( + HTTPError, + URLError, + ) from six.moves.urllib.parse import ( quote, urlparse, @@ -84,6 +88,7 @@ import transaction from webob.request import environ_from_url as orig_environ_from_url import wsgi_intercept from wsgi_intercept import httplib2_intercept +import zcml from zope.app.publication.httpfactory import HTTPPublicationRequestFactory from zope.app.wsgi import WSGIPublisherApplication from zope.component import ( @@ -155,7 +160,6 @@ from lp.testing import ( reset_logging, ) from lp.testing.pgsql import PgTestSetup -import zcml COMMA = ',' @@ -1794,6 +1798,14 @@ class LayerProcessController: raise LayerIsolationError( "App server died in this test (status=%s):\n%s" % ( cls.appserver.returncode, cls.appserver.stdout.read())) + # Cleanup the app server's output buffer between tests. + while True: + # Read while we have something available at the stdout. + r, w, e = select.select([cls.appserver.stdout], [], [], 0) + if cls.appserver.stdout in r: + cls.appserver.stdout.readline() + else: + break DatabaseLayer.force_dirty_database() @classmethod @@ -1836,6 +1848,12 @@ class LayerProcessController: try: connection = urlopen(root_url) connection.read() + except HTTPError as error: + if error.code == 503: + raise RuntimeError( + "App server is returning unknown error code %s. Is " + "there another instance running in the same port?" % + error.code) except URLError as error: # We are interested in a wrapped socket.error. if not isinstance(error.reason, socket.error):
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : [email protected] Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp
