Modified: trunk/Tools/BuildSlaveSupport/ews-build/steps.py (252445 => 252446)
--- trunk/Tools/BuildSlaveSupport/ews-build/steps.py 2019-11-14 02:26:55 UTC (rev 252445)
+++ trunk/Tools/BuildSlaveSupport/ews-build/steps.py 2019-11-14 03:07:35 UTC (rev 252446)
@@ -1046,7 +1046,7 @@
self.build.buildFinished([message], SUCCESS)
else:
self.setProperty('patchFailedTests', True)
- self.build.addStepsAfterCurrentStep([UnApplyPatchIfRequired(), CompileJSCToT(), RunJSCTestsWithoutPatch()])
+ self.build.addStepsAfterCurrentStep([UnApplyPatchIfRequired(), CompileJSCToT(), RunJSCTestsWithoutPatch(), AnalyzeJSCTestsResults()])
return rc
@@ -1058,6 +1058,82 @@
return shell.Test.evaluateCommand(self, cmd)
+class AnalyzeJSCTestsResults(buildstep.BuildStep):
+ name = 'analyze-jsc-tests-results'
+ description = ['analyze-jsc-test-results']
+ descriptionDone = ['analyze-jsc-tests-results']
+ NUM_FAILURES_TO_DISPLAY = 10
+
+ def start(self):
+ first_run_stress_failures = set(self.getProperty('jsc_stress_test_failures', []))
+ first_run_binary_failures = set(self.getProperty('jsc_binary_failures', []))
+ second_run_stress_failures = set(self.getProperty('jsc_rerun_stress_test_failures', []))
+ second_run_binary_failures = set(self.getProperty('jsc_rerun_binary_failures', []))
+ clean_tree_stress_failures = set(self.getProperty('jsc_clean_tree_stress_test_failures', []))
+ clean_tree_binary_failures = set(self.getProperty('jsc_clean_tree_binary_failures', []))
+ clean_tree_failures = list(clean_tree_binary_failures) + list(clean_tree_stress_failures)
+ clean_tree_failures_string = ', '.join(clean_tree_failures)
+
+ stress_failures_with_patch = first_run_stress_failures.intersection(second_run_stress_failures)
+ binary_failures_with_patch = first_run_binary_failures.intersection(second_run_binary_failures)
+
+ flaky_stress_failures = first_run_stress_failures.union(second_run_stress_failures) - first_run_stress_failures.intersection(second_run_stress_failures)
+ flaky_binary_failures = first_run_binary_failures.union(second_run_binary_failures) - first_run_binary_failures.intersection(second_run_binary_failures)
+ flaky_failures = list(flaky_binary_failures) + list(flaky_stress_failures)
+ flaky_failures_string = ', '.join(flaky_failures)
+
+ new_stress_failures = stress_failures_with_patch - clean_tree_stress_failures
+ new_binary_failures = binary_failures_with_patch - clean_tree_binary_failures
+ new_stress_failures_to_display = ', '.join(list(new_stress_failures)[:self.NUM_FAILURES_TO_DISPLAY])
+ new_binary_failures_to_display = ', '.join(list(new_binary_failures)[:self.NUM_FAILURES_TO_DISPLAY])
+
+ self._addToLog('stderr', '\nFailures in first run: {}'.format(list(first_run_binary_failures) + list(first_run_stress_failures)))
+ self._addToLog('stderr', '\nFailures in second run: {}'.format(list(second_run_binary_failures) + list(second_run_stress_failures)))
+ self._addToLog('stderr', '\nFlaky Tests: {}'.format(flaky_failures_string))
+ self._addToLog('stderr', '\nFailures on clean tree: {}'.format(list(clean_tree_stress_failures) + list(clean_tree_binary_failures)))
+
+ if new_stress_failures or new_binary_failures:
+ self._addToLog('stderr', '\nNew failures: {}\n'.format(list(new_binary_failures) + list(new_stress_failures)))
+ self.finished(FAILURE)
+ self.build.results = FAILURE
+ message = ''
+ if new_binary_failures:
+ pluralSuffix = 's' if len(new_binary_failures) > 1 else ''
+ message = 'Found {} new JSC binary failure{}: {}'.format(len(new_binary_failures), pluralSuffix, new_binary_failures_to_display)
+ if new_stress_failures:
+ if message:
+ message += ', '
+ pluralSuffix = 's' if len(new_stress_failures) > 1 else ''
+ message += 'Found {} new JSC stress test failure{}: {}'.format(len(new_stress_failures), pluralSuffix, new_stress_failures_to_display)
+ if len(new_stress_failures) > self.NUM_FAILURES_TO_DISPLAY:
+ message += ' ...'
+ self.descriptionDone = message
+ self.build.buildFinished([message], FAILURE)
+ else:
+ self._addToLog('stderr', '\nNo new failures\n')
+ self.finished(SUCCESS)
+ self.build.results = SUCCESS
+ self.descriptionDone = 'Passed JSC tests'
+ pluralSuffix = 's' if len(clean_tree_failures) > 1 else ''
+ message = ''
+ if clean_tree_failures:
+ message = 'Found {} pre-existing JSC test failure{}: {}'.format(len(clean_tree_failures), pluralSuffix, clean_tree_failures_string)
+ if len(clean_tree_failures) > self.NUM_FAILURES_TO_DISPLAY:
+ message += ' ...'
+ if flaky_failures:
+ message += ' Found flaky tests: {}'.format(flaky_failures_string)
+ self.build.buildFinished([message], SUCCESS)
+ return defer.succeed(None)
+
+ @defer.inlineCallbacks
+ def _addToLog(self, logName, message):
+ try:
+ log = self.getLog(logName)
+ except KeyError:
+ log = yield self.addLog(logName)
+ log.addStdout(message)
+
+
class CleanBuild(shell.Compile):
name = 'delete-WebKitBuild-directory'
description = ['deleting WebKitBuild directory']
Modified: trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py (252445 => 252446)
--- trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py 2019-11-14 02:26:55 UTC (rev 252445)
+++ trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py 2019-11-14 03:07:35 UTC (rev 252446)
@@ -34,7 +34,7 @@
from twisted.python import failure, log
from twisted.trial import unittest
-from steps import (AnalyzeAPITestsResults, AnalyzeCompileWebKitResults, AnalyzeLayoutTestsResults, ApplyPatch, ApplyWatchList, ArchiveBuiltProduct, ArchiveTestResults,
+from steps import (AnalyzeAPITestsResults, AnalyzeCompileWebKitResults, AnalyzeJSCTestsResults, AnalyzeLayoutTestsResults, ApplyPatch, ApplyWatchList, ArchiveBuiltProduct, ArchiveTestResults,
CheckOutSource, CheckOutSpecificRevision, CheckPatchRelevance, CheckStyle, CleanBuild, CleanUpGitIndexLock, CleanWorkingDirectory,
CompileJSC, CompileJSCToT, CompileWebKit, CompileWebKitToT, ConfigureBuild,
DownloadBuiltProduct, DownloadBuiltProductFromMaster, ExtractBuiltProduct, ExtractTestResults, InstallGtkDependencies, InstallWpeDependencies, KillOldProcesses,
@@ -1199,6 +1199,110 @@
return self.runStep()
+class TestAnalyzeJSCTestsResults(BuildStepMixinAdditions, unittest.TestCase):
+ def setUp(self):
+ self.longMessage = True
+ return self.setUpBuildStep()
+
+ def tearDown(self):
+ return self.tearDownBuildStep()
+
+ def configureStep(self):
+ self.setupStep(AnalyzeJSCTestsResults())
+ self.setProperty('jsc_stress_test_failures', [])
+ self.setProperty('jsc_binary_failures', [])
+ self.setProperty('jsc_rerun_stress_test_failures', [])
+ self.setProperty('jsc_rerun_binary_failures', [])
+ self.setProperty('jsc_clean_tree_stress_test_failures', [])
+ self.setProperty('jsc_clean_tree_binary_failures', [])
+
+ def test_single_new_stress_failure(self):
+ self.configureStep()
+ self.setProperty('jsc_stress_test_failures', ['stress/force-error.js.bytecode-cache'])
+ self.setProperty('jsc_rerun_stress_test_failures', ['stress/force-error.js.bytecode-cache'])
+ self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC stress test failure: stress/force-error.js.bytecode-cache (failure)')
+ return self.runStep()
+
+ def test_single_new_binary_failure(self):
+ self.configureStep()
+ self.setProperty('jsc_binary_failures', ['testmasm'])
+ self.setProperty('jsc_rerun_binary_failures', ['testmasm'])
+ self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC binary failure: testmasm (failure)')
+ return self.runStep()
+
+ def test_multiple_new_stress_failure(self):
+ self.configureStep()
+ self.setProperty('jsc_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)])
+ self.setProperty('jsc_rerun_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)])
+ self.expectOutcome(result=FAILURE, state_string='Found 30 new JSC stress test failures: test1, test0, test3, test2, test5, test4, test7, test6, test9, test8 ... (failure)')
+ return self.runStep()
+
+ def test_multiple_new_binary_failure(self):
+ self.configureStep()
+ self.setProperty('jsc_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi'])
+ self.setProperty('jsc_rerun_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi'])
+ self.expectOutcome(result=FAILURE, state_string='Found 5 new JSC binary failures: testb3, testmasm, testapi, testdfg, testair (failure)')
+ return self.runStep()
+
+ def test_new_stress_and_binary_failure(self):
+ self.configureStep()
+ self.setProperty('jsc_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default'])
+ self.setProperty('jsc_binary_failures', ['testmasm'])
+ self.setProperty('jsc_rerun_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default'])
+ self.setProperty('jsc_rerun_binary_failures', ['testmasm'])
+ self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC binary failure: testmasm, Found 1 new JSC stress test failure: es6.yaml/es6/Set_iterator_closing.js.default (failure)')
+ return self.runStep()
+
+ def test_stress_failure_on_clean_tree(self):
+ self.configureStep()
+ self.setProperty('jsc_stress_test_failures', ['stress/force-error.js.default'])
+ self.setProperty('jsc_rerun_stress_test_failures', ['stress/force-error.js.default'])
+ self.setProperty('jsc_clean_tree_stress_test_failures', ['stress/force-error.js.default'])
+ self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests')
+ return self.runStep()
+
+ def test_binary_failure_on_clean_tree(self):
+ self.configureStep()
+ self.setProperty('jsc_binary_failures', ['testdfg'])
+ self.setProperty('jsc_rerun_binary_failures', ['testdfg'])
+ self.setProperty('jsc_clean_tree_binary_failures', ['testdfg'])
+ self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests')
+ return self.runStep()
+
+ def test_stress_and_binary_failure_on_clean_tree(self):
+ self.configureStep()
+ self.setProperty('jsc_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default'])
+ self.setProperty('jsc_binary_failures', ['testair'])
+ self.setProperty('jsc_rerun_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default'])
+ self.setProperty('jsc_rerun_binary_failures', ['testair'])
+ self.setProperty('jsc_clean_tree_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default'])
+ self.setProperty('jsc_clean_tree_binary_failures', ['testair'])
+ self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests')
+ return self.runStep()
+
+ def test_flaky_stress_and_binary_failures(self):
+ self.configureStep()
+ self.setProperty('jsc_stress_test_failures', ['stress/force-error.js.default'])
+ self.setProperty('jsc_binary_failures', ['testapi'])
+ self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests')
+ return self.runStep()
+
+ def test_flaky_and_consistent_stress_failures(self):
+ self.configureStep()
+ self.setProperty('jsc_stress_test_failures', ['test1', 'test2'])
+ self.setProperty('jsc_rerun_stress_test_failures', ['test2'])
+ self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC stress test failure: test2 (failure)')
+ return self.runStep()
+
+ def test_flaky_and_consistent_failures_with_clean_tree_failures(self):
+ self.configureStep()
+ self.setProperty('jsc_stress_test_failures', ['test1', 'test2'])
+ self.setProperty('jsc_rerun_stress_test_failures', ['test1'])
+ self.setProperty('jsc_clean_tree_stress_test_failures', ['test1', 'test2'])
+ self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests')
+ return self.runStep()
+
+
class TestRunWebKitTests(BuildStepMixinAdditions, unittest.TestCase):
def setUp(self):
self.longMessage = True
