Modified: branches/safari-611-branch/Tools/ChangeLog (273363 => 273364)
--- branches/safari-611-branch/Tools/ChangeLog 2021-02-24 01:05:34 UTC (rev 273363)
+++ branches/safari-611-branch/Tools/ChangeLog 2021-02-24 01:05:36 UTC (rev 273364)
@@ -1,5 +1,64 @@
2021-02-23 Ryan Haddad <[email protected]>
+ Cherry-pick r272167. rdar://problem/74668621
+
+ autoinstall.py: Don't use an XML parser to parse HTML
+ https://bugs.webkit.org/show_bug.cgi?id=221162
+
+ Reviewed by Jonathan Bedard.
+
+ Very recently autoinstall.py started failing becagse the page returned
+ by https://pypi.org/simple/fasteners/ now contains a <meta> tag, which
+ is not parseable as XML without the trailing slash.
+
+ Despite what an initial fix in r272041 stated, this is not caused by
+ invalid HTML: <meta> tags are not required to have an ending slash to
+ be standards compliant in any version of HTML.
+
+ This patch replaces that code to use an HTML parser instead of an XML
+ parser, making it more robust.
+
+ * Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py:
+ (SimplyPypiIndexPageParser):
+ (SimplyPypiIndexPageParser.__init__):
+ (SimplyPypiIndexPageParser.handle_starttag):
+ (SimplyPypiIndexPageParser.handle_data):
+ (SimplyPypiIndexPageParser.handle_endtag):
+ (SimplyPypiIndexPageParser.parse):
+ (Package.archives):
+
+
+ git-svn-id: https://svn.webkit.org/repository/webkit/trunk@272167 268f45cc-cd09-0410-ab3c-d52691b4dbfc
+
+ 2021-02-01 Alicia Boya García <[email protected]>
+
+ autoinstall.py: Don't use an XML parser to parse HTML
+ https://bugs.webkit.org/show_bug.cgi?id=221162
+
+ Reviewed by Jonathan Bedard.
+
+ Very recently autoinstall.py started failing becagse the page returned
+ by https://pypi.org/simple/fasteners/ now contains a <meta> tag, which
+ is not parseable as XML without the trailing slash.
+
+ Despite what an initial fix in r272041 stated, this is not caused by
+ invalid HTML: <meta> tags are not required to have an ending slash to
+ be standards compliant in any version of HTML.
+
+ This patch replaces that code to use an HTML parser instead of an XML
+ parser, making it more robust.
+
+ * Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py:
+ (SimplyPypiIndexPageParser):
+ (SimplyPypiIndexPageParser.__init__):
+ (SimplyPypiIndexPageParser.handle_starttag):
+ (SimplyPypiIndexPageParser.handle_data):
+ (SimplyPypiIndexPageParser.handle_endtag):
+ (SimplyPypiIndexPageParser.parse):
+ (Package.archives):
+
+2021-02-23 Ryan Haddad <[email protected]>
+
Cherry-pick r272041. rdar://problem/74668621
[webkitcrepy] Handle case where pypi serves invalid html
Modified: branches/safari-611-branch/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py (273363 => 273364)
--- branches/safari-611-branch/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py 2021-02-24 01:05:34 UTC (rev 273363)
+++ branches/safari-611-branch/Tools/Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py 2021-02-24 01:05:36 UTC (rev 273364)
@@ -38,15 +38,45 @@
from logging import NullHandler
from webkitcorepy import log
from webkitcorepy.version import Version
-from xml.dom import minidom
if sys.version_info > (3, 0):
from urllib.request import urlopen
from urllib.error import URLError
+ from html.parser import HTMLParser
else:
from urllib2 import urlopen, URLError
+ from HTMLParser import HTMLParser
+class SimplyPypiIndexPageParser(HTMLParser):
+ def __init__(self):
+ HTMLParser.__init__(self)
+ self.packages = []
+ self.current_package = None
+
+ def handle_starttag(self, tag, attrs):
+ if tag == "a":
+ attrs_dict = dict(attrs)
+ if "href" not in attrs_dict:
+ return
+ self.current_package = attrs_dict
+
+ def handle_data(self, data):
+ if self.current_package is not None:
+ self.current_package["name"] = data
+
+ def handle_endtag(self, tag):
+ if tag == "a" and self.current_package is not None:
+ self.packages.append(self.current_package)
+ self.current_package = None
+
+ @staticmethod
+ def parse(html_text):
+ parser = SimplyPypiIndexPageParser()
+ parser.feed(html_text)
+ return parser.packages
+
+
class Package(object):
class Archive(object):
def __init__(self, name, link, version, extension=None):
@@ -126,27 +156,12 @@
if response.code != 200:
raise ValueError('The package {} was not found on {}'.format(self.pypi_name, AutoInstall.index))
- # In some cases, pypi serves invalid html
- content = response.read()
- content = re.sub(b'\s+<meta [^\n]+>\n', b'', content)
-
- page = minidom.parseString(content)
+ packages = SimplyPypiIndexPageParser.parse(response.read().decode("UTF-8"))
cached_tags = None
- for element in reversed(page.getElementsByTagName("a")):
- if not len(element.childNodes):
- continue
- if element.childNodes[0].nodeType != minidom.Node.TEXT_NODE:
- continue
-
- attributes = {}
- for index in range(element.attributes.length):
- attributes[element.attributes.item(index).name] = element.attributes.item(index).value
- if not attributes.get('href', None):
- continue
-
+ for package in reversed(packages):
if self.wheel:
- match = re.search(r'.+-([^-]+-[^-]+-[^-]+).whl', element.childNodes[0].data)
+ match = re.search(r'.+-([^-]+-[^-]+-[^-]+).whl', package['name'])
if not match:
continue
@@ -161,18 +176,18 @@
extension = 'whl'
else:
- if element.childNodes[0].data.endswith(('.tar.gz', '.tar.bz2')):
+ if package['name'].endswith(('.tar.gz', '.tar.bz2')):
extension = 'tar.gz'
- elif element.childNodes[0].data.endswith('.zip'):
+ elif package['name'].endswith('.zip'):
extension = 'zip'
else:
continue
- requires = attributes.get('data-requires-python')
+ requires = package.get('data-requires-python')
if requires and not AutoInstall.version.matches(requires):
continue
- version_candidate = re.search(r'\d+\.\d+(\.\d+)?', element.childNodes[0].data)
+ version_candidate = re.search(r'\d+\.\d+(\.\d+)?', package["name"])
if not version_candidate:
continue
version = Version(*version_candidate.group().split('.'))
@@ -179,7 +194,7 @@
if self.version and version not in self.version:
continue
- link = attributes['href'].split('#')[0]
+ link = package['href'].split('#')[0]
if '://' not in link:
depth = 0
while link.startswith('../'):
