Diff
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps_unittest.py (268929 => 268930)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps_unittest.py 2020-10-23 18:08:16 UTC (rev 268929)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps_unittest.py 2020-10-23 18:15:46 UTC (rev 268930)
@@ -45,7 +45,7 @@
def _add_dependent_modules_to_sys_modules(self):
self._add_webkitpy_to_sys_path()
- from webkitpy.thirdparty.autoinstalled import buildbot
+ from webkitpy.autoinstalled import buildbot
sys.modules['buildbot'] = buildbot
Modified: trunk/Tools/ChangeLog (268929 => 268930)
--- trunk/Tools/ChangeLog 2020-10-23 18:08:16 UTC (rev 268929)
+++ trunk/Tools/ChangeLog 2020-10-23 18:15:46 UTC (rev 268930)
@@ -1,3 +1,30 @@
+2020-10-23 Jonathan Bedard <[email protected]>
+
+ [webkitpy] Use webkitcorepy's autoinstaller for buildbot and twisted
+ https://bugs.webkit.org/show_bug.cgi?id=218105
+ <rdar://problem/70593576>
+
+ Rubber-stamped by Aakash Jain.
+
+ * BuildSlaveSupport/build.webkit.org-config/steps_unittest.py:
+ (BuildBotConfigLoader._add_dependent_modules_to_sys_modules):
+ * Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py: Bump version.
+ * Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py:
+ (Package.archives): tar.bz2 archives are tar.gz archives.
+ (Package.install): After r267056, we don't need "old-and-unmanageable" and
+ "single-version-externally-managed", handle case of nested modules more completely.
+ * Scripts/webkitpy/autoinstalled/buildbot.py: Added.
+ * Scripts/webkitpy/autoinstalled/twisted.py: Added.
+ * Scripts/webkitpy/benchmark_runner/http_server_driver/http_server/twisted_http_server.py:
+ Import new auto installed twisted.
+ * Scripts/webkitpy/benchmark_runner/http_server_driver/simple_http_server_driver.py:
+ (SimpleHTTPServerDriver._ensure_http_server_dependencies): Import new auto installed twisted.
+ * Scripts/webkitpy/thirdparty/__init__.py:
+ (AutoinstallImportHook.find_module): Remove all modules.
+ (AutoinstallImportHook._install_buildbot): Deleted.
+ (AutoinstallImportHook._install_twisted_15_5_0): Deleted.
+ * Scripts/webkitpy/thirdparty/__init___unittest.py: Removed.
+
2020-10-23 Aakash Jain <[email protected]>
Rename slave to worker in webkitpy
Modified: trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py (268929 => 268930)
--- trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py 2020-10-23 18:08:16 UTC (rev 268929)
+++ trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py 2020-10-23 18:15:46 UTC (rev 268930)
@@ -35,7 +35,7 @@
from webkitcorepy.subprocess_utils import TimeoutExpired, CompletedProcess, run
from webkitcorepy.output_capture import LoggerCapture, OutputCapture, OutputDuplicate
-version = Version(0, 4, 19)
+version = Version(0, 4, 20)
from webkitcorepy.autoinstall import Package, AutoInstall
if sys.version_info > (3, 0):
Modified: trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py (268929 => 268930)
--- trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py 2020-10-23 18:08:16 UTC (rev 268929)
+++ trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py 2020-10-23 18:15:46 UTC (rev 268930)
@@ -108,7 +108,7 @@
def location(self):
if not AutoInstall.directory:
raise ValueError('No AutoInstall directory, Package cannot resolve location')
- return os.path.join(AutoInstall.directory, self.name)
+ return os.path.join(AutoInstall.directory, *self.name.split('.'))
def do_post_install(self, archive_path):
pass
@@ -154,7 +154,7 @@
extension = 'whl'
else:
- if element.childNodes[0].data.endswith('.tar.gz'):
+ if element.childNodes[0].data.endswith(('.tar.gz', '.tar.bz2')):
extension = 'tar.gz'
elif element.childNodes[0].data.endswith('.zip'):
extension = 'zip'
@@ -224,7 +224,7 @@
archive = self.archives()[-1]
try:
- install_location = os.path.dirname(self.location)
+ install_location = AutoInstall.directory
shutil.rmtree(self.location, ignore_errors=True)
AutoInstall.log('Downloading {}...'.format(archive))
@@ -233,12 +233,8 @@
temp_location = os.path.join(tempfile.gettempdir(), self.name)
archive.unpack(temp_location)
- for candidate in [
- os.path.join(temp_location, str(archive)),
- os.path.join(temp_location, '{}-{}.{}'.format(archive.name, archive.version.major, archive.version.minor)),
- os.path.join(temp_location, '{}-{}.{}.{}'.format(archive.name.replace('-', '_'), archive.version.major, archive.version.minor, archive.version.tiny)),
- os.path.join(temp_location, '{}-{}'.format(archive.name.capitalize(), archive.version)),
- ]:
+ for candidate in os.listdir(temp_location):
+ candidate = os.path.join(temp_location, candidate)
if not os.path.exists(os.path.join(candidate, 'setup.py')):
continue
@@ -256,14 +252,10 @@
'--home={}'.format(install_location),
'--root=/',
'--prefix=',
- '--single-version-externally-managed',
'--install-lib={}'.format(install_location),
'--install-scripts={}'.format(os.path.join(install_location, 'bin')),
'--install-data="" 'data')),
'--install-headers={}'.format(os.path.join(install_location, 'headers')),
- # Do not automatically install package dependencies, force scripts to be explicit
- # Even without this flag, setup.py is not consistent about installing dependencies.
- '--old-and-unmanageable',
],
cwd=candidate,
env=dict(
@@ -278,6 +270,12 @@
stderr=devnull,
)
+ # If we have a package inside another package (like zope.interface), the top-level package needs an __init__.py
+ location = os.path.join(AutoInstall.directory, self.name.split('.')[0])
+ if os.path.isdir(location) and '__init__.py' not in os.listdir(location):
+ with open(os.path.join(location, '__init__.py'), 'w') as init:
+ init.write('\n')
+
break
else:
# We might not need setup.py at all, check if we have dist-info and the library in the temporary location
Added: trunk/Tools/Scripts/webkitpy/autoinstalled/buildbot.py (0 => 268930)
--- trunk/Tools/Scripts/webkitpy/autoinstalled/buildbot.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/autoinstalled/buildbot.py 2020-10-23 18:15:46 UTC (rev 268930)
@@ -0,0 +1,47 @@
+# Copyright (C) 2020 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+
+from webkitscmpy import AutoInstall, Package, Version
+
+if sys.version_info[0] >= 3:
+ AutoInstall.register(Package('autobahn', Version(20, 7, 1)))
+ AutoInstall.register(Package('buildbot', Version(2, 8, 4)))
+ AutoInstall.register(Package('dateutil', Version(2, 8, 1), pypi_name='python-dateutil'))
+ AutoInstall.register(Package('jinja2', Version(2, 11, 2), pypi_name='Jinja2'))
+ AutoInstall.register(Package('jwt', Version(1, 7, 1), pypi_name='PyJWT'))
+ AutoInstall.register(Package('pyyaml', Version(5, 3, 1), pypi_name='PyYAML'))
+ AutoInstall.register(Package('sqlalchemy', Version(1, 3, 20), pypi_name='SQLAlchemy'))
+ AutoInstall.register(Package('sqlalchemy-migrate', Version(0, 13, 0)))
+ AutoInstall.register(Package('twisted', Version(20, 3, 0), pypi_name='Twisted'))
+ AutoInstall.register(Package('txaio', Version(20, 4, 1)))
+
+else:
+ AutoInstall.register(Package('buildbot', Version(0, 8, 6)))
+ AutoInstall.register(Package('dateutil', Version(2, 8, 1), pypi_name='python-dateutil'))
+ AutoInstall.register(Package('jinja2', Version(2, 6), pypi_name='Jinja2'))
+ AutoInstall.register(Package('sqlalchemy', Version(0, 7, 7), pypi_name='SQLAlchemy'))
+ AutoInstall.register(Package('sqlalchemy-migrate', Version(0, 7, 2)))
+ AutoInstall.register(Package('twisted', Version(12, 1, 0), pypi_name='Twisted'))
+
+sys.modules[__name__] = __import__('buildbot')
Added: trunk/Tools/Scripts/webkitpy/autoinstalled/twisted.py (0 => 268930)
--- trunk/Tools/Scripts/webkitpy/autoinstalled/twisted.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/autoinstalled/twisted.py 2020-10-23 18:15:46 UTC (rev 268930)
@@ -0,0 +1,28 @@
+# Copyright (C) 2020 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+
+from webkitscmpy import AutoInstall, Package, Version
+
+AutoInstall.register(Package('twisted', Version(15, 5, 0), pypi_name='Twisted'))
+sys.modules[__name__] = __import__('twisted')
Modified: trunk/Tools/Scripts/webkitpy/benchmark_runner/http_server_driver/http_server/twisted_http_server.py (268929 => 268930)
--- trunk/Tools/Scripts/webkitpy/benchmark_runner/http_server_driver/http_server/twisted_http_server.py 2020-10-23 18:08:16 UTC (rev 268929)
+++ trunk/Tools/Scripts/webkitpy/benchmark_runner/http_server_driver/http_server/twisted_http_server.py 2020-10-23 18:15:46 UTC (rev 268930)
@@ -4,15 +4,10 @@
import logging
import os
import sys
+
from pkg_resources import require, VersionConflict, DistributionNotFound
+from webkitpy.autoinstalled import twisted
-try:
- require("Twisted>=15.5.0")
- import twisted
-except (ImportError, VersionConflict, DistributionNotFound):
- sys.path.append(os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../..')))
- from webkitpy.thirdparty.autoinstalled.twisted_15_5_0 import twisted
-
from twisted.web import static, server
from twisted.web.resource import Resource
from twisted.internet import reactor
Modified: trunk/Tools/Scripts/webkitpy/benchmark_runner/http_server_driver/simple_http_server_driver.py (268929 => 268930)
--- trunk/Tools/Scripts/webkitpy/benchmark_runner/http_server_driver/simple_http_server_driver.py 2020-10-23 18:08:16 UTC (rev 268929)
+++ trunk/Tools/Scripts/webkitpy/benchmark_runner/http_server_driver/simple_http_server_driver.py 2020-10-23 18:15:46 UTC (rev 268930)
@@ -105,10 +105,4 @@
def _ensure_http_server_dependencies(self):
_log.info('Ensure dependencies of http server is satisfied')
from pkg_resources import require, VersionConflict, DistributionNotFound
- try:
- require("Twisted>=15.5.0")
- import twisted
- except (ImportError, VersionConflict, DistributionNotFound):
- _log.info("Will install twisted in webkitpy, and twisted will be used by webkitpy only")
- sys.path.append(os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../../..')))
- from webkitpy.thirdparty.autoinstalled.twisted_15_5_0 import twisted
+ from webkitpy.autoinstalled import twisted
Modified: trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py (268929 => 268930)
--- trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py 2020-10-23 18:08:16 UTC (rev 268929)
+++ trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py 2020-10-23 18:15:46 UTC (rev 268930)
@@ -26,10 +26,8 @@
"""Autoinstalls third-party code required by WebKit."""
-import codecs
import json
import os
-import re
import sys
if sys.version_info > (3, 0):
@@ -81,47 +79,8 @@
sys.path.insert(sys.path.index(AutoInstall.directory) + 1 if AutoInstall.directory in sys.path else 0, _AUTOINSTALLED_DIR)
def find_module(self, fullname, path=None):
- # This method will run before each import. See http://www.python.org/dev/peps/pep-0302/
- if '.autoinstalled' not in fullname:
- return
+ pass
- # Note: all of the methods must follow the "_install_XXX" convention in
- # order for autoinstall_everything(), below, to work properly.
- if '.buildbot' in fullname:
- self._install_buildbot()
- elif '.twisted_15_5_0' in fullname:
- self._install_twisted_15_5_0()
-
- # autoinstalled.buildbot is used by BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py
- # and should ideally match the version of BuildBot used at build.webkit.org.
- def _install_buildbot(self):
- # The buildbot package uses jinja2, for example, in buildbot/status/web/base.py.
- # buildbot imports jinja2 directly (as though it were installed on the system),
- # so the search path needs to include jinja2. We put jinja2 in
- # its own directory so that we can include it in the search path
- # without including other modules as a side effect.
- jinja_dir = self._fs.join(_AUTOINSTALLED_DIR, "jinja2")
- installer = AutoInstaller(append_to_search_path=True, target_dir=jinja_dir)
- installer.install(url=""
- url_subpath="Jinja2-2.6/jinja2")
-
- SQLAlchemy_dir = self._fs.join(_AUTOINSTALLED_DIR, "sqlalchemy")
- installer = AutoInstaller(append_to_search_path=True, target_dir=SQLAlchemy_dir)
- installer.install(url=""
- url_subpath="SQLAlchemy-0.7.7/lib/sqlalchemy")
-
- twisted_dir = self._fs.join(_AUTOINSTALLED_DIR, "twisted")
- installer = AutoInstaller(prepend_to_search_path=True, target_dir=twisted_dir)
- installer.install(url="" url_subpath="Twisted-12.1.0/twisted")
-
- self._install("https://files.pythonhosted.org/packages/source/b/buildbot/buildbot-0.8.6p1.tar.gz", "buildbot-0.8.6p1/buildbot")
-
- def _install_twisted_15_5_0(self):
- twisted_dir = self._fs.join(_AUTOINSTALLED_DIR, "twisted_15_5_0")
- installer = AutoInstaller(prepend_to_search_path=True, target_dir=twisted_dir)
- installer.install(url="" url_subpath="Twisted-15.5.0/twisted")
- installer.install(url="" url_subpath="zope.interface-4.1.3/src/zope")
-
@staticmethod
def greater_than_equal_to_version(minimum, version):
for i in range(len(minimum.split('.'))):
Deleted: trunk/Tools/Scripts/webkitpy/thirdparty/__init___unittest.py (268929 => 268930)
--- trunk/Tools/Scripts/webkitpy/thirdparty/__init___unittest.py 2020-10-23 18:08:16 UTC (rev 268929)
+++ trunk/Tools/Scripts/webkitpy/thirdparty/__init___unittest.py 2020-10-23 18:15:46 UTC (rev 268930)
@@ -1,39 +0,0 @@
-# Copyright (C) 2011 Google Inc. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import sys
-import unittest
-
-from webkitpy.thirdparty import AutoinstallImportHook
-
-
-class ThirdpartyTest(unittest.TestCase):
- def test_imports(self):
- # This method tests that we can actually import everything.
- # unused-variable, import failures - pylint: disable-msg=W0612,E0611,F0401
- import webkitpy.thirdparty.autoinstalled.buildbot
