Diff
Modified: trunk/Tools/ChangeLog (213721 => 213722)
--- trunk/Tools/ChangeLog 2017-03-10 19:15:23 UTC (rev 213721)
+++ trunk/Tools/ChangeLog 2017-03-10 19:24:47 UTC (rev 213722)
@@ -1,3 +1,57 @@
+2017-03-10 Srinivasan Vijayaraghavan <svijayaragha...@apple.com>
+
+ Add support for Bindings EWS
+ https://bugs.webkit.org/show_bug.cgi?id=169308
+
+ Reviewed by Alexey Proskuryakov.
+
+ * Scripts/webkitpy/common/config/ews.json: Add bindings-ews. Can test without building; should_build = False.
+ * Scripts/webkitpy/common/net/bindingstestresults.py:
+ (BindingsTestResults.__init__): Removed 'errors'.
+ (BindingsTestResults.results_from_string): Ditto.
+ (BindingsTestResults.is_subset): Ditto.
+ (BindingsTestResults.equals): Ditto.
+ (BindingsTestResults.all_passed): Ditto.
+ (BindingsTestResults.failing_tests): Ditto.
+ * Scripts/webkitpy/common/net/bindingstestresults_unittest.py:
+ (BindingsTestResultsTest.test_results_from_string): Removed 'errors'.
+ (BindingsTestResultsTest.test_results_from_string_success): Removed 'errors'.
+ * Scripts/webkitpy/tool/bot/earlywarningsystemtask.py:
+ (EarlyWarningSystemTask.__init__): Add should_build parameter.
+ (EarlyWarningSystemTask.run): Check should_build before calling self._build().
+ * Scripts/webkitpy/tool/bot/retrylogic_unittest.py: Renamed from Tools/Scripts/webkitpy/tool/bot/jscews_unittest.py.
+ (MockBindingsEarlyWarningSystem): Based of MockJSCEarlyWarningSystem. Mocked so we can provide test results.
+ (MockBindingsEarlyWarningSystem.__init__): Sets attributes.
+ (MockBindingsEarlyWarningSystem.test_results): Returns test results provided by us, instead of JSON output.
+ (BindingsEarlyWarningSystemTest): Class to test retry logic in below situations.
+ (BindingsEarlyWarningSystemTest._results_indicate_all_passed): False if None or or has failures, else True.
+ (BindingsEarlyWarningSystemTest._create_task):
+ (BindingsEarlyWarningSystemTest.test_success_case): Clean patch on clean tree.
+ (BindingsEarlyWarningSystemTest.test_test_failure): Red patch on clean tree.
+ (BindingsEarlyWarningSystemTest.test_fix): Clean patch on red tree.
+ (BindingsEarlyWarningSystemTest.test_ineffective_patch): Red patch on red tree.
+ (BindingsEarlyWarningSystemTest.test_partially_effective_patch): Red patch on redder tree.
+ (BindingsEarlyWarningSystemTest.test_different_test_failures_in_patch_and_tree): Red patch on red tree.
+ * Scripts/webkitpy/tool/bot/patchanalysistask.py:
+ (PatchAnalysisTask._build_and_test_without_patch): Don't build if should_build is False (eg. on Bindings EWS).
+ (PatchAnalysisTask._retry_bindings_tests): Retry logic for Bindings tests.
+ (PatchAnalysisTask._test_patch): If tests failed on Bindings EWS, and call _retry_bindings_tests.
+ * Scripts/webkitpy/tool/commands/earlywarningsystem.py:
+ (AbstractEarlyWarningSystem.begin_work_queue): Use BindingsTestResultsReader on Bindings EWS.
+ (AbstractEarlyWarningSystem._create_task): Pass should_build argument.
+ (AbstractEarlyWarningSystem.load_ews_classes): Add should_build argument (True by default).
+ * Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py:
+ (TestBindingsEWS): Bindings EWS class, derived from AbstractEarlyWarningSystem.
+ (AbstractEarlyWarningSystemTest.test_failing_bindings_tests_message): Unit test message posted to Bugzilla.
+ (EarlyWarningSystemTest._default_expected_logs): Not building on all ewses anymore; make build line optional.
+ (EarlyWarningSystemTest.test_ews_name): Add bindings-ews.
+ * Scripts/webkitpy/tool/steps/checkpatchrelevance.py:
+ (CheckPatchRelevance): Add bindings paths.
+ (CheckPatchRelevance._changes_are_relevant): Cleaner way to check if we should perform this step.
+ * Scripts/webkitpy/tool/steps/steps_unittest.py: Add unit tests for Bindings EWS patch relevance.
+ (test_patch_relevant_bindings): Test for patch relevant to bindings.
+ (test_patch_not_relevant_bindings): Test for patch not relevant to bindings.
+
2017-03-09 Simon Fraser <simon.fra...@apple.com>
[iOS WK2] Layer content blurry with nested perspective and transforms
Modified: trunk/Tools/Scripts/webkitpy/common/config/ews.json (213721 => 213722)
--- trunk/Tools/Scripts/webkitpy/common/config/ews.json 2017-03-10 19:15:23 UTC (rev 213721)
+++ trunk/Tools/Scripts/webkitpy/common/config/ews.json 2017-03-10 19:24:47 UTC (rev 213722)
@@ -51,5 +51,12 @@
"name": "jsc-ews",
"group": "jsc",
"runTests": true
+ },
+ "Bindings EWS": {
+ "port": "mac",
+ "name": "bindings-ews",
+ "group": "bindings",
+ "runTests": true,
+ "shouldBuild": false
}
}
Modified: trunk/Tools/Scripts/webkitpy/common/net/bindingstestresults.py (213721 => 213722)
--- trunk/Tools/Scripts/webkitpy/common/net/bindingstestresults.py 2017-03-10 19:15:23 UTC (rev 213721)
+++ trunk/Tools/Scripts/webkitpy/common/net/bindingstestresults.py 2017-03-10 19:24:47 UTC (rev 213722)
@@ -28,12 +28,9 @@
class BindingsTestResults(AbstractTestResults):
- def __init__(self, failures, errors):
+ def __init__(self, failures):
self._failures = failures
- self._errors = errors
- self._failing_test_names = failures + errors
-
@classmethod
def results_from_string(cls, string):
parsed_results = cls.parse_json_string(string)
@@ -40,22 +37,22 @@
if not parsed_results:
return None
- if 'failures' not in parsed_results or 'errors' not in parsed_results:
+ if 'failures' not in parsed_results:
return None
- return cls(parsed_results['failures'], parsed_results['errors'])
+ return cls(parsed_results['failures'])
def is_subset(self, other):
- return set(self._failures) <= set(other._failures) and set(self._errors) <= set(other._errors)
+ return set(self._failures) <= set(other._failures)
def equals(self, other):
- return set(self._failures) == set(other._failures) and set(self._errors) == set(other._errors)
+ return set(self._failures) == set(other._failures)
def all_passed(self):
- return not self._failures and not self._errors
+ return not self._failures
def failing_tests(self):
- return self._failing_test_names
+ return self._failures
# No defined failure limit for bindings tests.
def did_exceed_test_failure_limit(self):
Modified: trunk/Tools/Scripts/webkitpy/common/net/bindingstestresults_unittest.py (213721 => 213722)
--- trunk/Tools/Scripts/webkitpy/common/net/bindingstestresults_unittest.py 2017-03-10 19:15:23 UTC (rev 213721)
+++ trunk/Tools/Scripts/webkitpy/common/net/bindingstestresults_unittest.py 2017-03-10 19:24:47 UTC (rev 213722)
@@ -27,17 +27,15 @@
class BindingsTestResultsTest(unittest.TestCase):
def test_results_from_string(self):
- incomplete_json_v1 = '{"failures": []}'
- incomplete_json_v2 = '{"errors":[]}'
- self.assertEqual(None, BindingsTestResults.results_from_string(incomplete_json_v1))
- self.assertEqual(None, BindingsTestResults.results_from_string(incomplete_json_v2))
+ incomplete_json = '{"key2": []}'
+ self.assertEqual(None, BindingsTestResults.results_from_string(incomplete_json))
def test_results_from_string_success(self):
- no_failures_string = '{"failures": [], "errors":[]}'
- no_failures_results = BindingsTestResults([], [])
+ no_failures_string = '{"failures": []}'
+ no_failures_results = BindingsTestResults([])
self.assertTrue(no_failures_results.equals(BindingsTestResults.results_from_string(no_failures_string)))
self.assertTrue(no_failures_results.all_passed())
- test_string = '{"failures": ["failure1"], "errors": ["error1", "error2"]}'
- test_results = BindingsTestResults(["failure1"], ["error1", "error2"])
+ test_string = '{"failures": ["failure1"]}'
+ test_results = BindingsTestResults(["failure1"])
self.assertTrue(test_results.equals(BindingsTestResults.results_from_string(test_string)))
Modified: trunk/Tools/Scripts/webkitpy/tool/bot/earlywarningsystemtask.py (213721 => 213722)
--- trunk/Tools/Scripts/webkitpy/tool/bot/earlywarningsystemtask.py 2017-03-10 19:15:23 UTC (rev 213721)
+++ trunk/Tools/Scripts/webkitpy/tool/bot/earlywarningsystemtask.py 2017-03-10 19:24:47 UTC (rev 213722)
@@ -35,9 +35,10 @@
class EarlyWarningSystemTask(PatchAnalysisTask):
- def __init__(self, delegate, patch, should_run_tests=True):
+ def __init__(self, delegate, patch, should_run_tests=True, should_build=True):
PatchAnalysisTask.__init__(self, delegate, patch)
self._should_run_tests = should_run_tests
+ self._should_build = should_build
def validate(self):
self._patch = self._delegate.refetch_patch(self._patch)
@@ -61,10 +62,11 @@
raise UnableToApplyPatch(self._patch)
if not self._check_patch_relevance():
raise PatchIsNotApplicable(self._patch)
- if not self._build():
- if not self._build_without_patch():
- return False
- return self.report_failure()
+ if self._should_build:
+ if not self._build():
+ if not self._build_without_patch():
+ return False
+ return self.report_failure()
if not self._should_run_tests:
return True
return self._test_patch()
Deleted: trunk/Tools/Scripts/webkitpy/tool/bot/jscews_unittest.py (213721 => 213722)
--- trunk/Tools/Scripts/webkitpy/tool/bot/jscews_unittest.py 2017-03-10 19:15:23 UTC (rev 213721)
+++ trunk/Tools/Scripts/webkitpy/tool/bot/jscews_unittest.py 2017-03-10 19:24:47 UTC (rev 213722)
@@ -1,218 +0,0 @@
-# Copyright (C) 2017 Apple Inc. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import logging
-import unittest
-
-from webkitpy.common.net.jsctestresults import JSCTestResults
-from webkitpy.common.system.executive import ScriptError
-from webkitpy.tool.bot.patchanalysistask import *
-from webkitpy.tool.commands.earlywarningsystem import AbstractEarlyWarningSystem
-from webkitpy.tool.mocktool import MockTool
-
-_log = logging.getLogger(__name__)
-
-
-class MockPatchAnalysisTask(PatchAnalysisTask):
- def __init__(self, delegate, patch, patches_passed_all_tests):
- self._delegate = delegate
- self._patch = patch
- self._script_error = None
- self._results_archive_from_patch_test_run = None
- self._results_from_patch_test_run = None
- self.error = None
- self._patches_passed_all_tests = patches_passed_all_tests
- self._test_run_count = 0
- self.failure_status_id = 0
-
- def _test(self):
- self._test_run_count += 1
-
- if self._patches_passed_all_tests.pop() == True:
- return True
- self._script_error = ScriptError('Regression test')
- return False
-
- def _build_and_test_without_patch(self):
- self._test_run_count += 1
- return True
-
- def validate(self):
- return True
-
- def test_run_count(self):
- return self._test_run_count
-
-
-# This is the delegate to be used with MockPatchAnalysisTask, above.
-class MockJSCEarlyWarningSystem(AbstractEarlyWarningSystem):
- def __init__(self, first_test_results, second_test_results, clean_test_results):
- AbstractEarlyWarningSystem.__init__(self)
- self._group = 'jsc'
- self._results_in_order = [clean_test_results, second_test_results, first_test_results]
-
- def test_results(self):
- return self._results_in_order.pop()
-
-
-class JSCEarlyWarningSystemTest(unittest.TestCase):
- def _results_indicate_all_passed(self, results):
- if results == None:
- return False
- return results.all_passed()
-
- def _create_task(self, first_test_results, second_test_results, clean_test_results):
- queue = MockJSCEarlyWarningSystem(first_test_results, second_test_results, clean_test_results)
- tool = MockTool(log_executive=True)
- patch = tool.bugs.fetch_attachment(10000)
- patches_passed_all_tests = map(self._results_indicate_all_passed, [second_test_results, first_test_results])
- return MockPatchAnalysisTask(queue, patch, patches_passed_all_tests)
-
- def test_success_case(self):
- first_test_results = JSCTestResults(True, [])
- second_test_results = JSCTestResults(True, [])
- clean_test_results = JSCTestResults(True, [])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- return_value = task._test_patch()
- self.assertEqual(task.test_run_count(), 1)
- self.assertTrue(return_value)
-
- def test_test_failure(self):
- first_test_results = JSCTestResults(True, ['Fail.js'])
- second_test_results = JSCTestResults(True, ['Fail.js'])
- clean_test_results = JSCTestResults(True, [])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- with self.assertRaises(ScriptError):
- return_value = task._test_patch()
- self.assertEqual(task.test_run_count(), 3)
-
- def test_fix(self):
- first_test_results = JSCTestResults(True, [])
- second_test_results = JSCTestResults(True, [])
- clean_test_results = JSCTestResults(True, ['Fail.js'])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- return_value = task._test_patch()
- self.assertEqual(task.test_run_count(), 1)
- self.assertTrue(return_value)
-
- def test_ineffective_patch(self):
- first_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
- second_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
- clean_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- return_value = task._test_patch()
- self.assertEqual(task.test_run_count(), 3)
- self.assertTrue(return_value)
-
- def test_partially_effective_patch(self):
- first_test_results = JSCTestResults(True, ['failure2.js'])
- second_test_results = JSCTestResults(True, ['failure2.js'])
- clean_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- return_value = task._test_patch()
- self.assertEqual(task.test_run_count(), 3)
- self.assertTrue(return_value)
-
- def test_different_test_failures_in_patch_and_tree(self):
- first_test_results = JSCTestResults(False, [])
- second_test_results = JSCTestResults(False, [])
- clean_test_results = JSCTestResults(True, ['failure1.js', 'failure2.js'])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- with self.assertRaises(ScriptError):
- return_value = task._test_patch()
- self.assertEqual(task.test_run_count(), 3)
-
- def test_first_results_could_not_be_read(self):
- first_test_results = None
- second_test_results = JSCTestResults(True, [])
- clean_test_results = JSCTestResults(True, [])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- return_value = task._test_patch()
- self.assertEqual(task.test_run_count(), 1)
- self.assertFalse(return_value)
-
- def test_second_results_could_not_be_read(self):
- first_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
- second_test_results = None
- clean_test_results = JSCTestResults(True, [])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- return_value = task._test_patch()
- self.assertEqual(task.test_run_count(), 2)
- self.assertFalse(return_value)
-
- def test_clean_results_could_not_be_read(self):
- first_test_results = JSCTestResults(True, ['failure2.js'])
- second_test_results = JSCTestResults(True, ['failure2.js'])
- clean_test_results = None
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- return_value = task._test_patch()
- self.assertEqual(task.test_run_count(), 3)
- self.assertFalse(return_value)
-
- def test_flaky_results_on_clean_tree_pass(self):
- first_test_results = JSCTestResults(True, ['failure2.js'])
- second_test_results = JSCTestResults(True, [])
- clean_test_results = JSCTestResults(True, [])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- return_value = task._test_patch()
- self.assertTrue(return_value)
- self.assertEqual(task.test_run_count(), 2)
-
- def test_flaky_results_on_clean_tree_pass_v2(self):
- first_test_results = JSCTestResults(True, [])
- second_test_results = JSCTestResults(True, ['failure2.js'])
- clean_test_results = JSCTestResults(True, [])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- return_value = task._test_patch()
- self.assertTrue(return_value)
- self.assertEqual(task.test_run_count(), 1)
-
- def test_flaky_results_on_clean_tree_failure(self):
- first_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
- second_test_results = JSCTestResults(True, ['failure2.js'])
- clean_test_results = JSCTestResults(True, [])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- with self.assertRaises(ScriptError):
- task._test_patch()
- self.assertEqual(task.test_run_count(), 3)
-
- def test_flaky_results_on_red_tree_pass(self):
- first_test_results = JSCTestResults(True, ['failure1.js'])
- second_test_results = JSCTestResults(True, ['failure1.js', 'failure2.js'])
- clean_test_results = JSCTestResults(True, ['failure1.js'])
- task = self._create_task(first_test_results, second_test_results, clean_test_results)
-
- return_value = task._test_patch()
- self.assertTrue(return_value)
- self.assertEqual(task.test_run_count(), 3)
Modified: trunk/Tools/Scripts/webkitpy/tool/bot/patchanalysistask.py (213721 => 213722)
--- trunk/Tools/Scripts/webkitpy/tool/bot/patchanalysistask.py 2017-03-10 19:15:23 UTC (rev 213721)
+++ trunk/Tools/Scripts/webkitpy/tool/bot/patchanalysistask.py 2017-03-10 19:24:47 UTC (rev 213722)
@@ -186,12 +186,14 @@
"build-and-test",
"--force-clean",
"--no-update",
- "--build",
"--test",
"--non-interactive",
"--build-style=%s" % self._delegate.build_style(),
]
+ if getattr(self._delegate, 'should_build', True):
+ args.append("--build")
+
if hasattr(self._delegate, 'group'):
args.append("--group=%s" % self._delegate.group())
@@ -235,6 +237,30 @@
# also present without the patch, so we don't need to defer.
return False
+ def _retry_bindings_tests(self):
+ first_results = self._delegate.test_results()
+ first_script_error = self._script_error
+ first_failure_status_id = self.failure_status_id
+ if first_results is None:
+ return False
+
+ # Some errors are not correctly reported by the run-bindings-tests script
+ # https://bugs.webkit.org/show_bug.cgi?id=169449
+ # In affected cases, add a message requesting to look at test output instead.
+ if not first_results._failures:
+ first_results._failures = ["Please see test output for results"]
+
+ self._build_and_test_without_patch()
+ clean_tree_results = self._delegate.test_results()
+ if clean_tree_results is None:
+ return False
+
+ if first_results.is_subset(clean_tree_results):
+ return True
+
+ self.failure_status_id = first_failure_status_id
+ return self.report_failure(None, first_results, first_script_error)
+
# FIXME: Abstract out common parts of the retry logic.
def _retry_jsc_tests(self):
first_results = self._delegate.test_results()
@@ -338,6 +364,8 @@
if hasattr(self._delegate, 'group') and self._delegate.group() == "jsc":
return self._retry_jsc_tests()
+ elif hasattr(self._delegate, 'group') and self._delegate.group() == "bindings":
+ return self._retry_bindings_tests()
else:
return self._retry_layout_tests()
Copied: trunk/Tools/Scripts/webkitpy/tool/bot/retrylogic_unittest.py (from rev 213719, trunk/Tools/Scripts/webkitpy/tool/bot/jscews_unittest.py) (0 => 213722)
--- trunk/Tools/Scripts/webkitpy/tool/bot/retrylogic_unittest.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/tool/bot/retrylogic_unittest.py 2017-03-10 19:24:47 UTC (rev 213722)
@@ -0,0 +1,297 @@
+# Copyright (C) 2017 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import logging
+import unittest
+
+from webkitpy.common.net.bindingstestresults import BindingsTestResults
+from webkitpy.common.net.jsctestresults import JSCTestResults
+from webkitpy.common.system.executive import ScriptError
+from webkitpy.tool.bot.patchanalysistask import *
+from webkitpy.tool.commands.earlywarningsystem import AbstractEarlyWarningSystem
+from webkitpy.tool.mocktool import MockTool
+
+_log = logging.getLogger(__name__)
+
+
+class MockPatchAnalysisTask(PatchAnalysisTask):
+ def __init__(self, delegate, patch, patches_passed_all_tests):
+ self._delegate = delegate
+ self._patch = patch
+ self._script_error = None
+ self._results_archive_from_patch_test_run = None
+ self._results_from_patch_test_run = None
+ self.error = None
+ self._patches_passed_all_tests = patches_passed_all_tests
+ self._test_run_count = 0
+ self.failure_status_id = 0
+
+ def _test(self):
+ self._test_run_count += 1
+
+ if self._patches_passed_all_tests.pop() == True:
+ return True
+ self._script_error = ScriptError('Regression test')
+ return False
+
+ def _build_and_test_without_patch(self):
+ self._test_run_count += 1
+ return True
+
+ def validate(self):
+ return True
+
+ def test_run_count(self):
+ return self._test_run_count
+
+
+# This is the delegate to be used with MockPatchAnalysisTask, above.
+class MockJSCEarlyWarningSystem(AbstractEarlyWarningSystem):
+ def __init__(self, first_test_results, second_test_results, clean_test_results):
+ AbstractEarlyWarningSystem.__init__(self)
+ self._group = 'jsc'
+ self._results_in_order = [clean_test_results, second_test_results, first_test_results]
+
+ def test_results(self):
+ return self._results_in_order.pop()
+
+
+class JSCEarlyWarningSystemTest(unittest.TestCase):
+ def _results_indicate_all_passed(self, results):
+ if results == None:
+ return False
+ return results.all_passed()
+
+ def _create_task(self, first_test_results, second_test_results, clean_test_results):
+ queue = MockJSCEarlyWarningSystem(first_test_results, second_test_results, clean_test_results)
+ tool = MockTool(log_executive=True)
+ patch = tool.bugs.fetch_attachment(10000)
+ patches_passed_all_tests = map(self._results_indicate_all_passed, [second_test_results, first_test_results])
+ return MockPatchAnalysisTask(queue, patch, patches_passed_all_tests)
+
+ def test_success_case(self):
+ first_test_results = JSCTestResults(True, [])
+ second_test_results = JSCTestResults(True, [])
+ clean_test_results = JSCTestResults(True, [])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 1)
+ self.assertTrue(return_value)
+
+ def test_test_failure(self):
+ first_test_results = JSCTestResults(True, ['Fail.js'])
+ second_test_results = JSCTestResults(True, ['Fail.js'])
+ clean_test_results = JSCTestResults(True, [])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ with self.assertRaises(ScriptError):
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 3)
+
+ def test_fix(self):
+ first_test_results = JSCTestResults(True, [])
+ second_test_results = JSCTestResults(True, [])
+ clean_test_results = JSCTestResults(True, ['Fail.js'])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 1)
+ self.assertTrue(return_value)
+
+ def test_ineffective_patch(self):
+ first_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
+ second_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
+ clean_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 3)
+ self.assertTrue(return_value)
+
+ def test_partially_effective_patch(self):
+ first_test_results = JSCTestResults(True, ['failure2.js'])
+ second_test_results = JSCTestResults(True, ['failure2.js'])
+ clean_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 3)
+ self.assertTrue(return_value)
+
+ def test_different_test_failures_in_patch_and_tree(self):
+ first_test_results = JSCTestResults(False, [])
+ second_test_results = JSCTestResults(False, [])
+ clean_test_results = JSCTestResults(True, ['failure1.js', 'failure2.js'])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ with self.assertRaises(ScriptError):
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 3)
+
+ def test_first_results_could_not_be_read(self):
+ first_test_results = None
+ second_test_results = JSCTestResults(True, [])
+ clean_test_results = JSCTestResults(True, [])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 1)
+ self.assertFalse(return_value)
+
+ def test_second_results_could_not_be_read(self):
+ first_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
+ second_test_results = None
+ clean_test_results = JSCTestResults(True, [])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 2)
+ self.assertFalse(return_value)
+
+ def test_clean_results_could_not_be_read(self):
+ first_test_results = JSCTestResults(True, ['failure2.js'])
+ second_test_results = JSCTestResults(True, ['failure2.js'])
+ clean_test_results = None
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 3)
+ self.assertFalse(return_value)
+
+ def test_flaky_results_on_clean_tree_pass(self):
+ first_test_results = JSCTestResults(True, ['failure2.js'])
+ second_test_results = JSCTestResults(True, [])
+ clean_test_results = JSCTestResults(True, [])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertTrue(return_value)
+ self.assertEqual(task.test_run_count(), 2)
+
+ def test_flaky_results_on_clean_tree_pass_v2(self):
+ first_test_results = JSCTestResults(True, [])
+ second_test_results = JSCTestResults(True, ['failure2.js'])
+ clean_test_results = JSCTestResults(True, [])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertTrue(return_value)
+ self.assertEqual(task.test_run_count(), 1)
+
+ def test_flaky_results_on_clean_tree_failure(self):
+ first_test_results = JSCTestResults(False, ['failure1.js', 'failure2.js'])
+ second_test_results = JSCTestResults(True, ['failure2.js'])
+ clean_test_results = JSCTestResults(True, [])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ with self.assertRaises(ScriptError):
+ task._test_patch()
+ self.assertEqual(task.test_run_count(), 3)
+
+ def test_flaky_results_on_red_tree_pass(self):
+ first_test_results = JSCTestResults(True, ['failure1.js'])
+ second_test_results = JSCTestResults(True, ['failure1.js', 'failure2.js'])
+ clean_test_results = JSCTestResults(True, ['failure1.js'])
+ task = self._create_task(first_test_results, second_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertTrue(return_value)
+ self.assertEqual(task.test_run_count(), 3)
+
+
+class MockBindingsEarlyWarningSystem(AbstractEarlyWarningSystem):
+ def __init__(self, first_test_results, clean_test_results):
+ AbstractEarlyWarningSystem.__init__(self)
+ self._group = 'bindings'
+ self._results_in_order = [clean_test_results, first_test_results]
+
+ def test_results(self):
+ return self._results_in_order.pop()
+
+
+class BindingsEarlyWarningSystemTest(unittest.TestCase):
+ def _results_indicate_all_passed(self, results):
+ if results == None:
+ return False
+ return results.all_passed()
+
+ def _create_task(self, first_test_results, clean_test_results):
+ queue = MockBindingsEarlyWarningSystem(first_test_results, clean_test_results)
+ tool = MockTool(log_executive=True)
+ patch = tool.bugs.fetch_attachment(10000)
+ patches_passed_all_tests = [self._results_indicate_all_passed(first_test_results)]
+ return MockPatchAnalysisTask(queue, patch, patches_passed_all_tests)
+
+ def test_success_case(self):
+ first_test_results = BindingsTestResults([])
+ clean_test_results = BindingsTestResults([])
+ task = self._create_task(first_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 1)
+ self.assertTrue(return_value)
+
+ def test_test_failure(self):
+ first_test_results = BindingsTestResults(['TestMapLike.idl'])
+ clean_test_results = BindingsTestResults([])
+ task = self._create_task(first_test_results, clean_test_results)
+
+ with self.assertRaises(ScriptError):
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 2)
+
+ def test_fix(self):
+ first_test_results = BindingsTestResults([])
+ clean_test_results = BindingsTestResults(['TestMapLike.idl'])
+ task = self._create_task(first_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 1)
+ self.assertTrue(return_value)
+
+ def test_ineffective_patch(self):
+ first_test_results = BindingsTestResults(['TestMapLike.idl'])
+ clean_test_results = BindingsTestResults(['TestMapLike.idl'])
+ task = self._create_task(first_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 2)
+ self.assertTrue(return_value)
+
+ def test_partially_effective_patch(self):
+ first_test_results = BindingsTestResults(['TestMapLike.idl'])
+ clean_test_results = BindingsTestResults(['TestMapLike.idl', 'TestNode.idl'])
+ task = self._create_task(first_test_results, clean_test_results)
+
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 2)
+ self.assertTrue(return_value)
+
+ def test_different_test_failures_in_patch_and_tree(self):
+ first_test_results = BindingsTestResults(['TestNode.idl'])
+ clean_test_results = BindingsTestResults(['TestMapLike.idl'])
+ task = self._create_task(first_test_results, clean_test_results)
+
+ with self.assertRaises(ScriptError):
+ return_value = task._test_patch()
+ self.assertEqual(task.test_run_count(), 2)
Modified: trunk/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem.py (213721 => 213722)
--- trunk/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem.py 2017-03-10 19:15:23 UTC (rev 213721)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem.py 2017-03-10 19:24:47 UTC (rev 213722)
@@ -37,6 +37,7 @@
from webkitpy.common.system.filesystem import FileSystem
from webkitpy.common.system.executive import ScriptError
from webkitpy.tool.bot.earlywarningsystemtask import EarlyWarningSystemTask, EarlyWarningSystemTaskDelegate
+from webkitpy.tool.bot.bindingstestresultsreader import BindingsTestResultsReader
from webkitpy.tool.bot.layouttestresultsreader import LayoutTestResultsReader
from webkitpy.tool.bot.jsctestresultsreader import JSCTestResultsReader
from webkitpy.tool.bot.patchanalysistask import UnableToApplyPatch, PatchIsNotValid, PatchIsNotApplicable
@@ -59,6 +60,8 @@
if self.group() == "jsc":
self._test_results_reader = JSCTestResultsReader(self._tool, self._port.jsc_results_directory())
+ elif self.group() == "bindings":
+ self._test_results_reader = BindingsTestResultsReader(self._tool, self._port.jsc_results_directory())
else:
self._test_results_reader = LayoutTestResultsReader(self._tool, self._port.results_directory(), self._log_directory())
@@ -88,7 +91,7 @@
# This exists for mocking
def _create_task(self, patch):
- return EarlyWarningSystemTask(self, patch, self._options.run_tests)
+ return EarlyWarningSystemTask(self, patch, should_run_tests=self._options.run_tests, should_build=self.should_build)
def review_patch(self, patch):
task = self._create_task(patch)
@@ -174,5 +177,6 @@
'watchers': config.get('watchers', []),
'run_tests': config.get('runTests', cls.run_tests),
'_group': config.get('group', None),
+ 'should_build': config.get('shouldBuild', True),
}))
return classes
Modified: trunk/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py (213721 => 213722)
--- trunk/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py 2017-03-10 19:15:23 UTC (rev 213721)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py 2017-03-10 19:24:47 UTC (rev 213722)
@@ -30,6 +30,7 @@
from webkitpy.thirdparty.mock import Mock
from webkitpy.common.host import Host
from webkitpy.common.host_mock import MockHost
+from webkitpy.common.net.bindingstestresults import BindingsTestResults
from webkitpy.common.net.jsctestresults import JSCTestResults
from webkitpy.common.net.layouttestresults import LayoutTestResults
from webkitpy.common.system.outputcapture import OutputCapture
@@ -56,6 +57,12 @@
_group = "jsc"
+class TestBindingsEWS(AbstractEarlyWarningSystem):
+ port_name = "mac"
+ _build_style = None
+ _group = "bindings"
+
+
class AbstractEarlyWarningSystemTest(QueuesTest):
def _test_message(self, ews, results, message):
ews.bind_to_tool(MockTool())
@@ -81,7 +88,13 @@
message = "New failing tests:\nes6.yaml/es6/typed_arrays_Int8Array.js.default\nes6.yaml/es6/typed_arrays_Uint8Array.js.default\napiTests"
self._test_message(ews, results, message)
+ def test_failing_bindings_tests_message(self):
+ ews = TestBindingsEWS()
+ results = lambda a: BindingsTestResults(["(JS) TestMapLike.idl", "(JS) TestNode.idl"])
+ message = "New failing tests:\n(JS) TestMapLike.idl\n(JS) TestNode.idl"
+ self._test_message(ews, results, message)
+
class MockEarlyWarningSystemTaskForInconclusiveJSCResults(EarlyWarningSystemTask):
def _test_patch(self):
self._test()
@@ -104,6 +117,13 @@
"build_style": ews.build_style(),
"group": ews.group(),
}
+
+ if ews.should_build:
+ build_line = "Running: webkit-patch --status-host=example.com build --no-clean --no-update --build-style=%(build_style)s --group=%(group)s --port=%(port)s%(architecture)s\n" % string_replacements
+ else:
+ build_line = ""
+ string_replacements['build_line'] = build_line
+
if ews.run_tests:
run_tests_line = "Running: webkit-patch --status-host=example.com build-and-test --no-clean --no-update --test --non-interactive --build-style=%(build_style)s --group=%(group)s --port=%(port)s%(architecture)s\n" % string_replacements
else:
@@ -123,8 +143,7 @@
Running: webkit-patch --status-host=example.com update --port=%(port)s%(architecture)s
Running: webkit-patch --status-host=example.com apply-attachment --no-update --non-interactive 10000 --port=%(port)s%(architecture)s
Running: webkit-patch --status-host=example.com check-patch-relevance --group=%(group)s --port=%(port)s%(architecture)s
-Running: webkit-patch --status-host=example.com build --no-clean --no-update --build-style=%(build_style)s --group=%(group)s --port=%(port)s%(architecture)s
-%(run_tests_line)s%(result_lines)s""" % string_replacements,
+%(build_line)s%(run_tests_line)s%(result_lines)s""" % string_replacements,
"handle_unexpected_error": "Mock error message\n",
"handle_script_error": "ScriptError error message\n\nMOCK output\n",
}
@@ -164,6 +183,7 @@
'mac-wk2-ews',
'mac-debug-ews',
'mac-32bit-ews',
+ 'bindings-ews',
'jsc-ews',
}
classes = AbstractEarlyWarningSystem.load_ews_classes()
Modified: trunk/Tools/Scripts/webkitpy/tool/steps/checkpatchrelevance.py (213721 => 213722)
--- trunk/Tools/Scripts/webkitpy/tool/steps/checkpatchrelevance.py 2017-03-10 19:15:23 UTC (rev 213721)
+++ trunk/Tools/Scripts/webkitpy/tool/steps/checkpatchrelevance.py 2017-03-10 19:24:47 UTC (rev 213722)
@@ -37,6 +37,11 @@
Options.group,
]
+ bindings_paths = [
+ "Source/WebCore",
+ "Tools",
+ ]
+
jsc_paths = [
"JSTests/",
"Source/_javascript_Core/",
@@ -52,12 +57,13 @@
]
group_to_paths_mapping = {
+ 'bindings': bindings_paths,
'jsc': jsc_paths,
}
def _changes_are_relevant(self, changed_files):
# In the default case, all patches are relevant
- if self._options.group != 'jsc':
+ if self._options.group not in self.group_to_paths_mapping:
return True
patterns = self.group_to_paths_mapping[self._options.group]
Modified: trunk/Tools/Scripts/webkitpy/tool/steps/steps_unittest.py (213721 => 213722)
--- trunk/Tools/Scripts/webkitpy/tool/steps/steps_unittest.py 2017-03-10 19:15:23 UTC (rev 213721)
+++ trunk/Tools/Scripts/webkitpy/tool/steps/steps_unittest.py 2017-03-10 19:24:47 UTC (rev 213722)
@@ -256,3 +256,29 @@
expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/run-bindings-tests', '--json-output=/tmp/bindings_test_results.json'], cwd=/mock-checkout
"""
OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
+
+ def test_patch_relevant_bindings(self):
+ self.maxDiff = None
+ mock_options = self._step_options()
+ mock_options.group = "bindings"
+ tool = MockTool(log_executive=True)
+ tool.scm()._mockChangedFiles = ["Source/WebCore/features.json", "Source/ChangeLog"]
+ # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
+ tool._deprecated_port = DeprecatedPort()
+ step = steps.CheckPatchRelevance(tool, mock_options)
+ expected_logs = """Checking relevance of patch
+"""
+ OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
+
+ def test_patch_not_relevant_bindings(self):
+ self.maxDiff = None
+ mock_options = self._step_options()
+ mock_options.group = "bindings"
+ tool = MockTool(log_executive=True)
+ tool.scm()._mockChangedFiles = ["Source/_javascript_Core/Makefile", "Source/ChangeLog"]
+ # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
+ tool._deprecated_port = DeprecatedPort()
+ step = steps.CheckPatchRelevance(tool, mock_options)
+ expected_logs = """Checking relevance of patch
+This patch does not have relevant changes.
+"""
