Modified: trunk/Tools/ChangeLog (265048 => 265049)
--- trunk/Tools/ChangeLog 2020-07-29 18:30:14 UTC (rev 265048)
+++ trunk/Tools/ChangeLog 2020-07-29 18:43:43 UTC (rev 265049)
@@ -1,5 +1,59 @@
2020-07-29 Jonathan Bedard <[email protected]>
+ [webkitcorepy] Add an auto-installer
+ https://bugs.webkit.org/show_bug.cgi?id=214606
+
+ Reviewed by Darin Adler.
+
+ Add an autoinstaller to webkitcorepy. This is eventually intended to replace the
+ autoinstaller in webkitpy.
+
+ * Scripts/libraries: Added property svn:ignore.
+ * Scripts/libraries/webkitcorepy/README.md:
+ * Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py:
+ * Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py: Added.
+ (Package): Class representing a single Python module.
+ (Package.Archive): Class representing a zip file containing a package.
+ (Package.Archive.__init__): Construct representation with a name, version and
+ downloadable link.
+ (Package.Archive.__repr__): Archives are named based on their package name
+ and version.
+ (Package.Archive.path): The location of a downloaded archive on disk.
+ (Package.Archive.download): Download archive and write contents to disk.
+ (Package.Archive.unpack): Unpack archive on disk into package.
+ (Package.__init__): Packages are constructed with a name, version and name on
+ PyPi, which is assumed to be the package name unless specified.
+ (Package.location): Location of the package on disk.
+ (Package.do_post_install): Some packages have post-install actions, execute
+ those.
+ (Package.archives): Return a list of archives which match the specified
+ package parameters.
+ (Package.is_cached): Check the AutoInstall manifest to see if a package has
+ already been installed.
+ (Package.install): If needed, find a compatible archive for the specified
+ package, download and install it.
+ (AutoInstall): Namespace for auto-install functionality.
+ (AutoInstall._request): Make a request to the package index specified.
+ (AutoInstall.enable): Enable the auto-installer.
+ (AutoInstall.disable): Disable the auto-installer.
+ (AutoInstall.set_directory): Set the directory to save auto-installed libraries to.
+ (AutoInstall.set_index): Use a different package index.
+ (AutoInstall.set_timeout): Set the default timeout.
+ (AutoInstall.register): Add a package to the list of packages to be installed
+ when encountered.
+ (AutoInstall.install): Given a package, register and install that package.
+ (AutoInstall.install_everything): Install all registered packages if they are not
+ already.
+ (AutoInstall.find_module): Check and see if a requested module has been registered,
+ if it has, install that module if needed.
+ * Scripts/libraries/webkitcorepy/webkitcorepy/cacert.pem: Added.
+ * Scripts/libraries/webkitcorepy/webkitcorepy/rootcerts.p7b: Added.
+ * Scripts/webkitpy/__init__.py: Add logger, autoinstalled requests and certifi.
+ * Scripts/webkitpy/test/main.py:
+ (Tester._run_tests): Include webkitcorepy.AutoInstall when installing everything.
+
+2020-07-29 Jonathan Bedard <[email protected]>
+
[build.webkit.org] Add Big Sur builder (Follow-up Fix)
https://bugs.webkit.org/show_bug.cgi?id=214879
<rdar://problem/66221071>
Modified: trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py (265048 => 265049)
--- trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py 2020-07-29 18:30:14 UTC (rev 265048)
+++ trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/__init__.py 2020-07-29 18:43:43 UTC (rev 265049)
@@ -20,7 +20,21 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import logging
+import sys
+
+log = logging.getLogger('webkitcorepy')
+
from webkitcorepy.version import Version
from webkitcorepy.string_utils import BytesIO, StringIO, UnicodeIO, unicode
-version = Version(0, 0, 2)
+version = Version(0, 0, 3)
+
+from webkitcorepy.autoinstall import Package, AutoInstall
+if sys.version_info > (3, 0):
+ AutoInstall.register(Package('mock', Version(4)))
+else:
+ AutoInstall.register(Package('mock', Version(3, 0, 5)))
+
+AutoInstall.register(Package('certifi', Version(2020, 6, 20)))
+AutoInstall.register(Package('requests', Version(2, 24)))
Added: trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py (0 => 265049)
--- trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py (rev 0)
+++ trunk/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py 2020-07-29 18:43:43 UTC (rev 265049)
@@ -0,0 +1,307 @@
+# Copyright (C) 2020 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import json
+import math
+import os
+import re
+import shutil
+import sys
+import tarfile
+
+from webkitcorepy import log
+from webkitcorepy.version import Version
+from xml.dom import minidom
+
+if sys.version_info > (3, 0):
+ from urllib.request import urlopen
+else:
+ from urllib2 import urlopen
+
+
+class Package(object):
+ class Archive(object):
+ def __init__(self, name, link, version):
+ self.name = name
+ self.link = link
+ self.version = version
+
+ def __repr__(self):
+ return '{}-{}.{}.{}'.format(self.name, self.version.major, self.version.minor, self.version.tiny)
+
+ @property
+ def path(self):
+ if not AutoInstall.directory:
+ raise ValueError('No AutoInstall directory, archive cannot resolve local path')
+ return '{}/{}-{}.tar.gz'.format(AutoInstall.directory, self.name, self.version)
+
+ def download(self):
+ response = AutoInstall._request(self.link)
+ try:
+ if response.code != 200:
+ raise IOError('Failed to retrieve Python module with response code {}'.format(response.code))
+ with open(self.path, 'wb') as file:
+ while True:
+ data = "" ** 13)
+ if not data:
+ break
+ file.write(data)
+ finally:
+ response.close()
+
+ def unpack(self, target):
+ if not os.path.isfile(self.path):
+ raise IOError('Failed to find archive at {}'.format(self.path))
+ shutil.rmtree(target, ignore_errors=True)
+ file = tarfile.open(self.path)
+ try:
+ file.extractall(target)
+ finally:
+ file.close()
+
+ def __init__(self, name, version=None, pypi_name=None):
+ self.name = name
+ self.version = version
+ self._archives = []
+ self.pypi_name = pypi_name or self.name
+
+ @property
+ def location(self):
+ if not AutoInstall.directory:
+ raise ValueError('No AutoInstall directory, Package cannot resolve location')
+ return os.path.join(AutoInstall.directory, self.name)
+
+ def do_post_install(self, archive_path):
+ pass
+
+ def archives(self):
+ if self._archives:
+ return self._archives
+
+ path = 'simple/{}/'.format(self.pypi_name)
+ response = AutoInstall._request('https://{}/{}'.format(AutoInstall.index, path))
+ try:
+ if response.code != 200:
+ raise ValueError('The package {} was not found on {}'.format(self.pypi_name, AutoInstall.index))
+
+ page = minidom.parseString(response.read())
+ for element in page.getElementsByTagName("a"):
+ if not len(element.childNodes):
+ continue
+ if element.childNodes[0].nodeType != minidom.Node.TEXT_NODE:
+ continue
+
+ attributes = {}
+ for index in range(element.attributes.length):
+ attributes[element.attributes.item(index).name] = element.attributes.item(index).value
+ if not attributes.get('href', None):
+ continue
+ if not element.childNodes[0].data.endswith('tar.gz'):
+ continue
+
+ requires = attributes.get('data-requires-python')
+ if requires and not AutoInstall.version.matches(requires):
+ continue
+
+ version_candidate = re.search(r'\d+\.\d+\.\d+', element.childNodes[0].data)
+ if not version_candidate:
+ continue
+ version = Version(*version_candidate.group().split('.'))
+ if self.version and version not in self.version:
+ continue
+
+ link = attributes['href'].split('#')[0]
+ if '://' not in link:
+ depth = 0
+ while link.startswith('../'):
+ depth += 1
+ link = link[3:]
+ link = 'https://{}/{}{}'.format(AutoInstall.index, '/'.join(path.split('/')[depth:]), link)
+
+ self._archives.append(self.Archive(
+ name=self.pypi_name,
+ link=link,
+ version=version,
+ ))
+
+ self._archives = sorted(self._archives, key=lambda archive: archive.version)
+ return self._archives
+ finally:
+ response.close()
+
+ def is_cached(self):
+ manifest = AutoInstall.manifest.get(self.name)
+ if not manifest:
+ return False
+ if manifest.get('index') != AutoInstall.index:
+ return False
+ if not manifest.get('version'):
+ return False
+ return not self.version or Version(*manifest.get('version').split('.')) in self.version
+
+ def install(self):
+ AutoInstall.register(self)
+ if self.is_cached():
+ return
+
+ if not self.archives():
+ raise ValueError('No archives for {}-{} found'.format(self.pypi_name, self.version))
+ archive = self.archives()[-1]
+
+ try:
+ log.warning('Installing {}...'.format(archive))
+ archive.download()
+
+ temp_location = '{}.tmp'.format(self.location)
+ archive.unpack(temp_location)
+
+ shutil.rmtree(self.location, ignore_errors=True)
+ shutil.move(os.path.join(temp_location, str(archive), self.name), self.location)
+
+ self.do_post_install(temp_location)
+
+ os.remove(archive.path)
+ shutil.rmtree(temp_location, ignore_errors=True)
+
+ AutoInstall.manifest[self.name] = {
+ 'index': AutoInstall.index,
+ 'version': str(archive.version),
+ }
+ with open(os.path.join(AutoInstall.directory, 'manifest.json'), 'w') as file:
+ json.dump(AutoInstall.manifest, file)
+
+ log.warning('Installed {}!'.format(archive))
+ except Exception:
+ log.critical('Failed to install {}!'.format(archive))
+ raise
+
+
+class AutoInstall(object):
+ _enabled = None
+ enabled = False
+ directory = None
+ index = 'pypi.org'
+ timeout = 30
+ version = Version(sys.version_info[0], sys.version_info[1], sys.version_info[2])
+ packages = {}
+ manifest = {}
+
+ @classmethod
+ def _request(cls, url):
+ # Rely on our own certificates for PyPi, since we use PyPi to standardize root certificates
+ if url.startswith('https://pypi.org') or url.startswith('https://files.pythonhosted.org'):
+ return urlopen(
+ url,
+ timeout=cls.timeout,
+ cafile=os.path.join(os.path.dirname(__file__), 'cacert.pem'),
+ )
+ return urlopen(url, timeout=cls.timeout)
+
+ @classmethod
+ def enable(cls):
+ cls._enabled = True
+ cls.enabled = True
+
+ @classmethod
+ def disable(cls):
+ cls._enabled = False
+ cls.enabled = False
+
+ @classmethod
+ def set_directory(cls, directory):
+ if not directory or not isinstance(directory, str):
+ raise ValueError('{} is an invalid autoinstall directory'.format(directory))
+
+ directory = os.path.abspath(directory)
+ if not os.path.isdir(directory):
+ if os.path.exists(directory):
+ raise ValueError('{} is not a directory and cannot be used as the autoinstall location')
+ os.makedirs(directory)
+
+ try:
+ with open(os.path.join(directory, 'manifest.json'), 'r') as file:
+ cls.manifest = json.load(file)
+ except (IOError, OSError, ValueError):
+ pass
+
+ sys.path.insert(0, directory)
+ cls.directory = directory
+ if cls._enabled is None:
+ cls._enabled = True
+ cls.enabled = True
+
+ @classmethod
+ def set_index(cls, index, check=True):
+ if check:
+ response = AutoInstall._request('https://{}/simple/pip/'.format(index))
+ try:
+ if response.code != 200:
+ raise ValueError('Failed to set AutoInstall index to {}, received {} response when searching for pip'.format(index, response.code))
+ finally:
+ response.close()
+ cls.index = index
+
+ @classmethod
+ def set_timeout(cls, timeout):
+ if timeout is not None and timeout <= 0:
+ raise ValueError('{} is an invalid timeout value'.format(timeout))
+ cls.timeout = math.ceil(timeout)
+
+ @classmethod
+ def register(cls, package):
+ if isinstance(package, str):
+ if cls.packages.get(package):
+ return cls.packages[package]
+ else:
+ package = Package(package)
+ elif isinstance(package, Package):
+ if cls.packages.get(package.name):
+ if cls.packages.get(package.name).version != package.version:
+ raise ValueError('Registered version of {} uses {}, but requested version uses {}'.format(package.name, cls.packages.get(package.name).version, package.version))
+ return cls.packages.get(package.name)
+ else:
+ raise ValueError('Expected package to be str or Package, not {}'.format(type(package)))
+ cls.packages[package.name] = package
+ return package
+
+ @classmethod
+ def install(cls, package):
+ to_install = cls.register(package)
+ return to_install.install()
+
+ @classmethod
+ def install_everything(cls):
+ for package in cls.packages.values():
+ package.install()
+ return None
+
+ @classmethod
+ def find_module(cls, fullname, path=None):
+ if not cls.enabled or path is not None:
+ return
+
+ name = fullname.split('.')[0]
+ if cls.packages.get(name):
+ cls.install(name)
+
+
+sys.meta_path.insert(0, AutoInstall)