Diff
Modified: trunk/Tools/ChangeLog (243029 => 243030)
--- trunk/Tools/ChangeLog 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/ChangeLog 2019-03-16 02:48:57 UTC (rev 243030)
@@ -1,3 +1,55 @@
+2019-03-15 Jonathan Bedard <[email protected]>
+
+ webkitpy: Upload test results
+ https://bugs.webkit.org/show_bug.cgi?id=195755
+ <rdar://problem/48896182>
+
+ Reviewed by Aakash Jain.
+
+ Establish a new format for uploading results that is not tied to layout tests, apply
+ that format to webkitpy tests.
+
+ * Scripts/webkitpy/common/checkout/scm/git.py:
+ (Git.native_branch): Return what branch the current checkout is on.
+ * Scripts/webkitpy/common/checkout/scm/scm_mock.py:
+ * Scripts/webkitpy/common/checkout/scm/scm_unittest.py:
+ * Scripts/webkitpy/common/checkout/scm/svn.py:
+ (SVN.native_branch): Ditto.
+ * Scripts/webkitpy/common/system/platforminfo.py:
+ (PlatformInfo.build_version): Return a build version for Mac.
+ * Scripts/webkitpy/common/system/platforminfo_mock.py:
+ (MockPlatformInfo.__init__):
+ (MockPlatformInfo.build_version):
+ * Scripts/webkitpy/results: Added.
+ * Scripts/webkitpy/results/__init__.py: Added.
+ * Scripts/webkitpy/results/options.py: Added.
+ (upload_options): OptParse list for upload options.
+ * Scripts/webkitpy/results/upload.py: Added.
+ (Upload): Class which enforces the upload format expected by the results server.
+ (Upload.Expectations):
+ (Upload.create_configuration):
+ (Upload.create_commit):
+ (Upload.create_details):
+ (Upload.create_run_stats):
+ (Upload.create_test_result):
+ (Upload.__init__):
+ (Upload.Encoder): Encode Upload object as json.
+ (Upload.upload): Upload results to the results server, returning 'True' if the upload is successful.
+ * Scripts/webkitpy/results/upload_unittest.py: Added.
+ * Scripts/webkitpy/test/main.py:
+ (Tester._parse_args): Add upload arguments.
+ (Tester._run_tests): Allow results to be uploaded.
+ * Scripts/webkitpy/test/runner.py:
+ (Runner.__init__): Record which tests were run, rather than just counting them.
+ (Runner.handle):
+ * Scripts/webkitpy/test/runner_unittest.py:
+ (RunnerTest.test_run):
+ * Scripts/webkitpy/thirdparty/__init__.py:
+ (AutoinstallImportHook.find_module): Add requests auto-install.
+ (AutoinstallImportHook._install_requests):
+ * Scripts/webkitpy/tool/commands/queues_unittest.py:
+ (PatchProcessingQueueTest.test_upload_results_archive_for_patch): Update os name for testing.
+
2019-03-15 Wenson Hsieh <[email protected]>
[iOS] Crash under -[TestRunnerWKWebView _didShowMenu] while running layout tests on iOS simulator
Modified: trunk/Tools/Scripts/webkitpy/common/checkout/scm/git.py (243029 => 243030)
--- trunk/Tools/Scripts/webkitpy/common/checkout/scm/git.py 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/Scripts/webkitpy/common/checkout/scm/git.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -290,6 +290,14 @@
def native_revision(self, path):
return self._run_git(['-C', self.find_checkout_root(path), 'log', '-1', '--pretty=format:%H'])
+ def native_branch(self, path):
+ result = self._run_git(['-C', self.find_checkout_root(path), 'rev-parse', '--abbrev-ref', 'HEAD']).rstrip()
+
+ # For git-svn
+ if result.startswith('heads'):
+ return result[6:]
+ return result
+
def svn_url(self):
git_command = ['svn', 'info']
status = self._run_git(git_command)
Modified: trunk/Tools/Scripts/webkitpy/common/checkout/scm/scm_mock.py (243029 => 243030)
--- trunk/Tools/Scripts/webkitpy/common/checkout/scm/scm_mock.py 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/Scripts/webkitpy/common/checkout/scm/scm_mock.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -87,6 +87,9 @@
def native_revision(self, path):
return self.svn_revision(path)
+ def native_branch(self, path):
+ return 'trunk'
+
def timestamp_of_revision(self, path, revision):
return '2013-02-01 08:48:05 +0000'
Modified: trunk/Tools/Scripts/webkitpy/common/checkout/scm/scm_unittest.py (243029 => 243030)
--- trunk/Tools/Scripts/webkitpy/common/checkout/scm/scm_unittest.py 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/Scripts/webkitpy/common/checkout/scm/scm_unittest.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -909,6 +909,9 @@
self.assertEqual(self.scm.head_svn_revision(), self.scm.native_revision('.'))
self.assertEqual(self.scm.native_revision('.'), '5')
+ def test_native_branch(self):
+ self.assertEqual(self.scm.native_branch('.'), 'trunk')
+
def test_propset_propget(self):
filepath = os.path.join(self.svn_checkout_path, "test_file")
expected_mime_type = "x-application/foo-bar"
@@ -1112,6 +1115,10 @@
command = ['git', '-C', scm.checkout_root, 'rev-parse', 'HEAD']
self.assertEqual(scm.native_revision(scm.checkout_root), run_command(command).strip())
+ def test_native_branch(self):
+ scm = self.tracking_scm
+ self.assertEqual('master', scm.native_branch(scm.checkout_root))
+
def test_rename_files(self):
scm = self.tracking_scm
@@ -1627,6 +1634,9 @@
command = ['git', '-C', self.git_checkout_path, 'rev-parse', 'HEAD']
self.assertEqual(self.scm.native_revision(self.git_checkout_path), run_command(command).strip())
+ def test_native_branch(self):
+ self.assertEqual('trunk', self.scm.native_branch(self.git_checkout_path))
+
def test_to_object_name(self):
relpath = 'test_file_commit1'
fullpath = os.path.realpath(os.path.join(self.git_checkout_path, relpath))
Modified: trunk/Tools/Scripts/webkitpy/common/checkout/scm/svn.py (243029 => 243030)
--- trunk/Tools/Scripts/webkitpy/common/checkout/scm/svn.py 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/Scripts/webkitpy/common/checkout/scm/svn.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -83,7 +83,7 @@
SCM.__init__(self, cwd, **kwargs)
self._bogus_dir = None
if patch_directories == []:
- raise Exception(message='Empty list of patch directories passed to SCM.__init__')
+ raise Exception('Empty list of patch directories passed to SCM.__init__')
elif patch_directories == None:
self._patch_directories = [self._filesystem.relpath(cwd, self.checkout_root)]
else:
@@ -278,6 +278,14 @@
def native_revision(self, path):
return self.svn_revision(path)
+ def native_branch(self, path):
+ relative_url = self.value_from_svn_info(path, 'Relative URL')[2:]
+ if relative_url.startswith('trunk'):
+ return 'trunk'
+ elif relative_url.startswith('branch'):
+ return relative_url.split('/')[1]
+ raise Exception('{} is not a branch'.format(relative_url.split('/')[0]))
+
def timestamp_of_revision(self, path, revision):
# We use --xml to get timestamps like 2013-02-08T08:18:04.964409Z
repository_root = self.value_from_svn_info(self.checkout_root, 'Repository Root')
Modified: trunk/Tools/Scripts/webkitpy/common/system/platforminfo.py (243029 => 243030)
--- trunk/Tools/Scripts/webkitpy/common/system/platforminfo.py 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/Scripts/webkitpy/common/system/platforminfo.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -146,6 +146,11 @@
except:
return sys.maxint
+ def build_version(self):
+ if self.is_mac():
+ return self._executive.run_command(['/usr/bin/sw_vers', '-buildVersion'], return_stderr=False, ignore_errors=True).rstrip()
+ return None
+
def xcode_sdk_version(self, sdk_name):
if self.is_mac():
# Assumes that xcrun does not write to standard output on failure (e.g. SDK does not exist).
Modified: trunk/Tools/Scripts/webkitpy/common/system/platforminfo_mock.py (243029 => 243030)
--- trunk/Tools/Scripts/webkitpy/common/system/platforminfo_mock.py 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/Scripts/webkitpy/common/system/platforminfo_mock.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -32,7 +32,7 @@
class MockPlatformInfo(object):
- def __init__(self, os_name='mac', os_version=Version.from_name('Snow Leopard')):
+ def __init__(self, os_name='mac', os_version=Version.from_name('High Sierra')):
assert isinstance(os_version, Version)
self.os_name = os_name
self.os_version = os_version
@@ -78,6 +78,11 @@
def terminal_width(self):
return 80
+ def build_version(self):
+ if self.is_mac():
+ return '17A405'
+ return None
+
def xcode_sdk_version(self, sdk_name):
return Version(8, 1)
Added: trunk/Tools/Scripts/webkitpy/results/__init__.py (0 => 243030)
--- trunk/Tools/Scripts/webkitpy/results/__init__.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/results/__init__.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -0,0 +1,13 @@
+# Required for Python to search this directory for module files
+
+# Keep this file free of any code or import statements that could
+# cause either an error to occur or a log message to be logged.
+# This ensures that calling code can import initialization code from
+# webkitpy before any errors or log messages due to code in this file.
+# Initialization code can include things like version-checking code and
+# logging configuration code.
+#
+# We do not execute any version-checking code or logging configuration
+# code in this file so that callers can opt-in as they want. This also
+# allows different callers to choose different initialization code,
+# as necessary.
Added: trunk/Tools/Scripts/webkitpy/results/options.py (0 => 243030)
--- trunk/Tools/Scripts/webkitpy/results/options.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/results/options.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -0,0 +1,33 @@
+# Copyright (C) 2019 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import optparse
+
+
+def upload_options():
+ return [
+ optparse.make_option('--report', action='', dest='report_urls', help='URL (or URLs) to report test results to'),
+ optparse.make_option('--buildbot-master', help='The url of the buildbot master.'),
+ optparse.make_option('--builder-name', help='The name of the buildbot builder tests were run on.'),
+ optparse.make_option('--build-number', help='The buildbot build number tests are associated with.'),
+ optparse.make_option('--buildbot-worker', help='The buildbot worker tests were run on.'),
+ ]
Added: trunk/Tools/Scripts/webkitpy/results/upload.py (0 => 243030)
--- trunk/Tools/Scripts/webkitpy/results/upload.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/results/upload.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -0,0 +1,184 @@
+# Copyright (C) 2019 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import webkitpy.thirdparty.autoinstalled.requests
+
+import json
+import requests
+import sys
+
+import platform as host_platform
+
+
+class Upload(object):
+ UPLOAD_ENDPOINT = '/api/upload'
+ BUILDBOT_DETAILS = ['buildbot-master', 'builder-name', 'build-number', 'buildbot-worker']
+ VERSION = 0
+
+ class Expectations:
+ # These are ordered by priority, meaning that a test which both crashes and has
+ # a warning should be considered to have crashed.
+ ORDER = [
+ 'CRASH',
+ 'TIMEOUT',
+ 'IMAGE', # Image-diff
+ 'AUDIO', # Audio-diff
+ 'TEXT', # Text-diff
+ 'FAIL',
+ 'ERROR',
+ 'WARNING',
+ 'PASS',
+ ]
+ CRASH, TIMEOUT, IMAGE, AUDIO, TEXT, FAIL, ERROR, WARNING, PASS = ORDER
+
+ class Encoder(json.JSONEncoder):
+
+ def default(self, obj):
+ if not isinstance(obj, Upload):
+ return super(Upload.Encoder, self).default(obj)
+
+ if not obj.suite:
+ raise ValueError('No suite specified to results upload')
+ if not obj.commits:
+ raise ValueError('No commits specified to results upload')
+
+ details = obj.details or obj.create_details()
+ buildbot_args = [details.get(arg, None) is None for arg in obj.BUILDBOT_DETAILS]
+ if any(buildbot_args) and not all(buildbot_args):
+ raise ValueError('All buildbot details must be defined for upload, details missing: {}'.format(', '.join(
+ [obj.BUILDBOT_DETAILS[i] for i in xrange(len(obj.BUILDBOT_DETAILS)) if buildbot_args[i]],
+ )))
+
+ def unpack_test(current, path_to_test, data):
+ if len(path_to_test) == 1:
+ current[path_to_test[0]] = data
+ return
+ if not current.get(path_to_test[0]):
+ current[path_to_test[0]] = {}
+ unpack_test(current[path_to_test[0]], path_to_test[1:], data)
+
+ results = {}
+ for test, data in obj.results.iteritems():
+ unpack_test(results, test.split('/'), data)
+
+ result = dict(
+ version=obj.VERSION,
+ suite=obj.suite,
+ configuration=obj.configuration or obj.create_configuration(),
+ commits=obj.commits,
+ test_results=dict(
+ details=details,
+ run_stats=obj.run_stats or obj.create_run_stats(),
+ results=results,
+ ),
+ )
+ if obj.timestamp:
+ result['timestamp'] = obj.timestamp
+ return result
+
+ def __init__(self, suite=None, configuration=None, commits=[], timestamp=None, details=None, run_stats=None, results={}):
+ self.suite = suite
+ self.configuration = configuration
+ self.commits = commits
+ self.timestamp = timestamp
+ self.details = details
+ self.run_stats = run_stats
+ self.results = results
+
+ @staticmethod
+ def create_configuration(
+ platform=None,
+ is_simulator=False,
+ version=None,
+ architecture=None,
+ version_name=None,
+ model=None,
+ style=None, # Debug/Production/Release
+ flavor=None, # Dumping ground suite-wide configuration changes (ie, GuardMalloc)
+ sdk=None,
+ ):
+
+ # This deviates slightly from the rest of webkitpy, but it allows this file to be entirely portable.
+ config = dict(
+ platform=platform or (host_platform.system() if host_platform.system() != 'Darwin' else 'mac').lower(),
+ is_simulator=is_simulator,
+ version=version or (host_platform.release() if host_platform.system() != 'Darwin' else host_platform.mac_ver()[0]),
+ architecture=architecture or host_platform.machine(),
+ )
+ optional_data = dict(version_name=version_name, model=model, style=style, flavor=flavor, sdk=sdk)
+ config.update({key: value for key, value in optional_data.iteritems() if value is not None})
+ return config
+
+ @staticmethod
+ def create_commit(repository_id, id, branch=None):
+ commit = dict(repository_id=repository_id, id=id)
+ if branch:
+ commit['branch'] = branch
+ return commit
+
+ @staticmethod
+ def create_details(link=None, options=None, **kwargs):
+ result = dict(**kwargs)
+ if link:
+ result['link'] = link
+ if not options:
+ return result
+
+ for element in Upload.BUILDBOT_DETAILS:
+ value = getattr(options, element.replace('-', '_'), None)
+ if value is not None:
+ result[element] = value
+ return result
+
+ @staticmethod
+ def create_run_stats(start_time=None, end_time=None, tests_skipped=None, **kwargs):
+ stats = dict(**kwargs)
+ optional_data = dict(start_time=start_time, end_time=end_time, tests_skipped=tests_skipped)
+ stats.update({key: value for key, value in optional_data.iteritems() if value is not None})
+ return stats
+
+ @staticmethod
+ def create_test_result(expected=None, actual=None, log=None, **kwargs):
+ result = dict(**kwargs)
+
+ # Tests which don't declare expectations or results are assumed to have passed.
+ optional_data = dict(expected=expected, actual=actual, log=log)
+ result.update({key: value for key, value in optional_data.iteritems() if value is not None})
+ return result
+
+ def upload(self, url, log_line_func=lambda val: sys.stdout.write(val + '\n')):
+ try:
+ response = requests.post(url + self.UPLOAD_ENDPOINT, data="" cls=Upload.Encoder))
+ except requests.exceptions.ConnectionError:
+ log_line_func(' ' * 4 + 'Failed to upload to {}, results server not online'.format(url))
+ return False
+ except ValueError as e:
+ log_line_func(' ' * 4 + 'Failed to encode upload data: {}'.format(e))
+ return False
+
+ if response.status_code != 200:
+ log_line_func(' ' * 4 + 'Error uploading to {}:'.format(url))
+ log_line_func(' ' * 8 + response.json()['description'])
+ return False
+
+ log_line_func(' ' * 4 + 'Uploaded results to {}'.format(url))
+ return True
Added: trunk/Tools/Scripts/webkitpy/results/upload_unittest.py (0 => 243030)
--- trunk/Tools/Scripts/webkitpy/results/upload_unittest.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/results/upload_unittest.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -0,0 +1,216 @@
+# Copyright (C) 2019 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import webkitpy.thirdparty.autoinstalled.requests
+
+import collections
+import json
+import requests
+import time
+import unittest
+
+from webkitpy.results.upload import Upload
+from webkitpy.thirdparty import mock
+
+
+class UploadTest(unittest.TestCase):
+
+ class Options(object):
+ def __init__(self, **kwargs):
+ for key, value in kwargs.iteritems():
+ setattr(self, key, value)
+
+ class MockResponse(object):
+ def __init__(self, status_code=200, text=''):
+ self.status_code = status_code
+ self.text = text
+
+ def json(self):
+ return json.loads(self.text)
+
+ @staticmethod
+ def normalize(data):
+ if isinstance(data, basestring):
+ return str(data)
+ elif isinstance(data, collections.Mapping):
+ return dict(map(UploadTest.normalize, data.iteritems()))
+ elif isinstance(data, collections.Iterable):
+ return type(data)(map(UploadTest.normalize, data))
+ return data
+
+ @staticmethod
+ def raise_requests_ConnectionError():
+ raise requests.exceptions.ConnectionError()
+
+ def test_encoding(self):
+ start_time, end_time = time.time() - 3, time.time()
+ upload = Upload(
+ suite='webkitpy-tests',
+ configuration=Upload.create_configuration(
+ platform='mac',
+ version='10.13.0',
+ version_name='High Sierra',
+ architecture='x86_64',
+ sdk='17A405',
+ ),
+ details=Upload.create_details(link='https://webkit.org'),
+ commits=[Upload.create_commit(
+ repository_id='webkit',
+ id='5',
+ branch='trunk',
+ )],
+ run_stats=Upload.create_run_stats(
+ start_time=start_time,
+ end_time=end_time,
+ tests_skipped=0,
+ ),
+ results={
+ 'webkitpy.test1': {},
+ 'webkitpy.test2': Upload.create_test_result(expected=Upload.Expectations.PASS, actual=Upload.Expectations.FAIL),
+ },
+ )
+ generated_dict = self.normalize(json.loads(json.dumps(upload, cls=Upload.Encoder)))
+
+ self.assertEqual(generated_dict['version'], 0)
+ self.assertEqual(generated_dict['suite'], 'webkitpy-tests')
+ self.assertEqual(generated_dict['configuration'], self.normalize(dict(
+ platform='mac',
+ is_simulator=False,
+ version='10.13.0',
+ version_name='High Sierra',
+ architecture='x86_64',
+ sdk='17A405',
+ )))
+ self.assertEqual(generated_dict['commits'], [dict(
+ repository_id='webkit',
+ id='5',
+ branch='trunk',
+ )])
+ self.assertEqual(generated_dict['test_results']['details'], self.normalize(dict(link='https://webkit.org')))
+ self.assertEqual(generated_dict['test_results']['run_stats'], self.normalize(dict(
+ start_time=start_time,
+ end_time=end_time,
+ tests_skipped=0,
+ )))
+ self.assertEqual(generated_dict['test_results']['results'], self.normalize({
+ 'webkitpy.test1': {},
+ 'webkitpy.test2': Upload.create_test_result(expected=Upload.Expectations.PASS, actual=Upload.Expectations.FAIL),
+ }))
+
+ def test_upload(self):
+ upload = Upload(
+ suite='webkitpy-tests',
+ commits=[Upload.create_commit(
+ repository_id='webkit',
+ id='5',
+ branch='trunk',
+ )],
+ )
+
+ with mock.patch('requests.post', new=lambda url, data: self.MockResponse()):
+ self.assertTrue(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))
+
+ with mock.patch('requests.post', new=lambda url, data: self.raise_requests_ConnectionError()):
+ self.assertFalse(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))
+
+ mock_404 = mock.patch('requests.post', new=lambda url, data: self.MockResponse(
+ status_code=404,
+ text=json.dumps(dict(description='No such address')),
+ ))
+ with mock_404:
+ self.assertFalse(upload.upload('https://webkit.org/results', log_line_func=lambda _: None))
+
+ def test_packed_test(self):
+ upload = Upload(
+ suite='webkitpy-tests',
+ commits=[Upload.create_commit(
+ repository_id='webkit',
+ id='5',
+ branch='trunk',
+ )],
+ results={
+ 'dir1/sub-dir1/test1': Upload.create_test_result(actual=Upload.Expectations.FAIL),
+ 'dir1/sub-dir1/test2': Upload.create_test_result(actual=Upload.Expectations.TIMEOUT),
+ 'dir1/sub-dir2/test3': {},
+ 'dir1/sub-dir2/test4': {},
+ 'dir2/sub-dir3/test5': {},
+ 'dir2/test6': {},
+ }
+ )
+ generated_dict = self.normalize(json.loads(json.dumps(upload, cls=Upload.Encoder)))
+ self.assertEqual(generated_dict['test_results']['results'], self.normalize({
+ 'dir1': {
+ 'sub-dir1': {
+ 'test1': {'actual': Upload.Expectations.FAIL},
+ 'test2': {'actual': Upload.Expectations.TIMEOUT},
+ }, 'sub-dir2': {
+ 'test3': {},
+ 'test4': {},
+ }
+ }, 'dir2': {
+ 'sub-dir3': {'test5': {}},
+ 'test6': {},
+ },
+ }))
+
+ def test_no_suite(self):
+ upload = Upload(
+ commits=[Upload.create_commit(
+ repository_id='webkit',
+ id='5',
+ branch='trunk',
+ )],
+ )
+ with self.assertRaises(ValueError):
+ json.dumps(upload, cls=Upload.Encoder)
+
+ def test_no_commits(self):
+ upload = Upload(
+ suite='webkitpy-tests',
+ )
+ with self.assertRaises(ValueError):
+ json.dumps(upload, cls=Upload.Encoder)
+
+ def test_buildbot(self):
+ upload = Upload(
+ suite='webkitpy-tests',
+ commits=[Upload.create_commit(
+ repository_id='webkit',
+ id='5',
+ branch='trunk',
+ )],
+ details=Upload.create_details(options=self.Options(
+ buildbot_master='webkit.org',
+ builder_name='Queue-1',
+ build_number=1,
+ )))
+ with self.assertRaises(ValueError):
+ json.dumps(upload, cls=Upload.Encoder)
+
+ upload.details['buildbot-worker'] = 'bot123'
+ generated_dict = self.normalize(json.loads(json.dumps(upload, cls=Upload.Encoder)))
+ self.assertEqual(generated_dict['test_results']['details'], self.normalize({
+ 'buildbot-master': 'webkit.org',
+ 'builder-name': 'Queue-1',
+ 'build-number': 1,
+ 'buildbot-worker': 'bot123',
+ }))
Modified: trunk/Tools/Scripts/webkitpy/test/main.py (243029 => 243030)
--- trunk/Tools/Scripts/webkitpy/test/main.py 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/Scripts/webkitpy/test/main.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -1,6 +1,6 @@
# Copyright (C) 2012 Google, Inc.
# Copyright (C) 2010 Chris Jerdonek ([email protected])
-# Copyright (C) 2018 Apple Inc. All rights reserved.
+# Copyright (C) 2018-2019 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
@@ -38,17 +38,19 @@
import unittest
from webkitpy.common.system.logutils import configure_logging
-from webkitpy.common.system.executive import Executive, ScriptError
+from webkitpy.common.system.executive import ScriptError
from webkitpy.common.system.filesystem import FileSystem
-from webkitpy.common.system.systemhost import SystemHost
+from webkitpy.common.host import Host
from webkitpy.port.config import Config
from webkitpy.test.finder import Finder
from webkitpy.test.printer import Printer
from webkitpy.test.runner import Runner, unit_test_name
+from webkitpy.results.upload import Upload
+from webkitpy.results.options import upload_options
_log = logging.getLogger(__name__)
-_host = SystemHost()
+_host = Host()
_webkit_root = None
@@ -131,6 +133,10 @@
help='Set the configuration to Release')
parser.add_option_group(configuration_group)
+ upload_group = optparse.OptionGroup(parser, 'Upload Options')
+ upload_group.add_options(upload_options())
+ parser.add_option_group(upload_group)
+
parser.add_option('-a', '--all', action='', default=False,
help='run all the tests')
parser.add_option('-c', '--coverage', action='', default=False,
@@ -181,11 +187,12 @@
from webkitpy.thirdparty import autoinstall_everything
autoinstall_everything()
+ start_time = time.time()
+ config = Config(_host.executive, self.finder.filesystem)
+ configuration_to_use = self._options.configuration or config.default_configuration()
if will_run_lldb_webkit_tests:
self.printer.write_update('Building lldbWebKitTester ...')
build_lldbwebkittester = self.finder.filesystem.join(_webkit_root, 'Tools', 'Scripts', 'build-lldbwebkittester')
- config = Config(_host.executive, self.finder.filesystem)
- configuration_to_use = self._options.configuration or config.default_configuration()
try:
_host.executive.run_and_throw_if_fail([build_lldbwebkittester, config.flag_for_configuration(configuration_to_use)], quiet=(not bool(self._options.verbose)))
except ScriptError as e:
@@ -218,6 +225,7 @@
test_runner = Runner(self.printer, loader)
test_runner.run(parallel_tests, self._options.child_processes)
test_runner.run(serial_tests, 1)
+ end_time = time.time()
self.printer.print_result(time.time() - start)
@@ -232,9 +240,51 @@
if self._options.coverage:
cov.stop()
cov.save()
+
+ failed_uploads = 0
+ if self._options.report_urls:
+ self.printer.meter.writeln('\n')
+ self.printer.write_update('Preparing upload data ...')
+
+ # Empty test results indicate a PASS.
+ results = {test: {} for test in test_runner.tests_run}
+ for test, errors in test_runner.errors:
+ results[test] = Upload.create_test_result(actual=Upload.Expectations.ERROR, log='/n'.join(errors))
+ for test, failures in test_runner.failures:
+ results[test] = Upload.create_test_result(actual=Upload.Expectations.FAIL, log='/n'.join(failures))
+
+ _host.initialize_scm()
+ upload = Upload(
+ suite='webkitpy-tests',
+ configuration=Upload.create_configuration(
+ platform=_host.platform.os_name,
+ version=str(_host.platform.os_version),
+ version_name=_host.platform.os_version_name(),
+ style=configuration_to_use,
+ sdk=_host.platform.build_version(),
+ ),
+ details=Upload.create_details(options=self._options),
+ commits=[Upload.create_commit(
+ repository_id='webkit',
+ id=_host.scm().native_revision(_webkit_root),
+ branch=_host.scm().native_branch(_webkit_root),
+ )],
+ run_stats=Upload.create_run_stats(
+ start_time=start_time,
+ end_time=end_time,
+ tests_skipped=len(test_runner.tests_run) - len(parallel_tests) - len(serial_tests),
+ ),
+ results=results,
+ )
+ for url in self._options.report_urls:
+ self.printer.write_update('Uploading to {} ...'.format(url))
+ failed_uploads = failed_uploads if upload.upload(url, log_line_func=self.printer.meter.writeln) else (failed_uploads + 1)
+ self.printer.meter.writeln('Uploads completed!')
+
+ if self._options.coverage:
cov.report(show_missing=False)
- return not self.printer.num_errors and not self.printer.num_failures
+ return not self.printer.num_errors and not self.printer.num_failures and not failed_uploads
def _check_imports(self, names):
for name in names:
Modified: trunk/Tools/Scripts/webkitpy/test/runner.py (243029 => 243030)
--- trunk/Tools/Scripts/webkitpy/test/runner.py 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/Scripts/webkitpy/test/runner.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -40,7 +40,7 @@
def __init__(self, printer, loader):
self.printer = printer
self.loader = loader
- self.tests_run = 0
+ self.tests_run = []
self.errors = []
self.failures = []
self.worker_factory = lambda caller: _Worker(caller, self.loader)
@@ -60,7 +60,7 @@
self.printer.print_started_test(source, test_name)
return
- self.tests_run += 1
+ self.tests_run.append(test_name)
if failures:
self.failures.append((test_name, failures))
if errors:
Modified: trunk/Tools/Scripts/webkitpy/test/runner_unittest.py (243029 => 243030)
--- trunk/Tools/Scripts/webkitpy/test/runner_unittest.py 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/Scripts/webkitpy/test/runner_unittest.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -92,6 +92,6 @@
('test3 (Foo)', 'E', 'test3\nerred'))
runner = Runner(Printer(stream, options), loader)
runner.run(['Foo.test1', 'Foo.test2', 'Foo.test3'], 1)
- self.assertEqual(runner.tests_run, 3)
+ self.assertEqual(len(runner.tests_run), 3)
self.assertEqual(len(runner.failures), 1)
self.assertEqual(len(runner.errors), 1)
Modified: trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py (243029 => 243030)
--- trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -1,4 +1,5 @@
# Copyright (C) 2010 Chris Jerdonek ([email protected])
+# Copyright (C) 2019 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
@@ -114,6 +115,8 @@
self._install_pytest_timeout()
elif '.pytest' in fullname:
self._install_pytest()
+ elif '.requests' in fullname:
+ self._install_requests()
def _install_mechanize(self):
self._install("https://files.pythonhosted.org/packages/source/m/mechanize/mechanize-0.2.5.tar.gz",
@@ -161,6 +164,21 @@
self._install("https://files.pythonhosted.org/packages/a2/ec/415d0cccc1ed41cd7fdf69ad989da16a8d13057996371004cab4bafc48f3/pytest-3.6.2.tar.gz",
"pytest-3.6.2/src/pytest.py")
+ def _install_requests(self):
+ self._ensure_autoinstalled_dir_is_in_sys_path()
+ self._install("https://files.pythonhosted.org/packages/06/b8/d1ea38513c22e8c906275d135818fee16ad8495985956a9b7e2bb21942a1/certifi-2019.3.9.tar.gz",
+ "certifi-2019.3.9/certifi")
+ self._install("https://files.pythonhosted.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz",
+ "chardet-3.0.4/chardet")
+ self._install("https://files.pythonhosted.org/packages/fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d/chardet-3.0.4.tar.gz",
+ "chardet-3.0.4/chardet")
+ self._install("https://files.pythonhosted.org/packages/ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7/idna-2.8.tar.gz",
+ "idna-2.8/idna")
+ self._install("https://files.pythonhosted.org/packages/b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1/urllib3-1.24.1.tar.gz",
+ "urllib3-1.24.1/src/urllib3")
+ self._install("https://files.pythonhosted.org/packages/52/2c/514e4ac25da2b08ca5a464c50463682126385c4272c18193876e91f4bc38/requests-2.21.0.tar.gz",
+ "requests-2.21.0/requests")
+
def _install_pylint(self):
self._ensure_autoinstalled_dir_is_in_sys_path()
if (not self._fs.exists(self._fs.join(_AUTOINSTALLED_DIR, "pylint")) or
Modified: trunk/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py (243029 => 243030)
--- trunk/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py 2019-03-16 02:21:45 UTC (rev 243029)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py 2019-03-16 02:48:57 UTC (rev 243030)
@@ -183,10 +183,10 @@
queue._options = Mock()
queue._options.port = None
patch = queue._tool.bugs.fetch_attachment(10001)
- expected_logs = """MOCK add_attachment_to_bug: bug_id=50000, description=Archive of layout-test-results from bot for mac-snowleopard filename=layout-test-results.zip mimetype=None
+ expected_logs = """MOCK add_attachment_to_bug: bug_id=50000, description=Archive of layout-test-results from bot for mac-highsierra filename=layout-test-results.zip mimetype=None
-- Begin comment --
The attached test failures were seen while running run-webkit-tests on the mock-queue.
-Port: mac-snowleopard Platform: MockPlatform 1.0
+Port: mac-highsierra Platform: MockPlatform 1.0
-- End comment --
"""
OutputCapture().assert_outputs(self, queue._upload_results_archive_for_patch, [patch, Mock()], expected_logs=expected_logs)
