Modified: trunk/Tools/CISupport/ews-build/steps.py (281760 => 281761)
--- trunk/Tools/CISupport/ews-build/steps.py 2021-08-30 15:56:19 UTC (rev 281760)
+++ trunk/Tools/CISupport/ews-build/steps.py 2021-08-30 16:02:14 UTC (rev 281761)
@@ -1961,7 +1961,9 @@
prefix = 'jsc_clean_tree_'
def evaluateCommand(self, cmd):
- return shell.Test.evaluateCommand(self, cmd)
+ rc = shell.Test.evaluateCommand(self, cmd)
+ self.setProperty('clean_tree_run_status', rc)
+ return rc
class AnalyzeJSCTestsResults(buildstep.BuildStep):
@@ -1990,8 +1992,8 @@
new_stress_failures = stress_failures_with_patch - clean_tree_stress_failures
new_binary_failures = binary_failures_with_patch - clean_tree_binary_failures
- new_stress_failures_to_display = ', '.join(sorted(list(new_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY])
- new_binary_failures_to_display = ', '.join(sorted(list(new_binary_failures))[:self.NUM_FAILURES_TO_DISPLAY])
+ self.new_stress_failures_to_display = ', '.join(sorted(list(new_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY])
+ self.new_binary_failures_to_display = ', '.join(sorted(list(new_binary_failures))[:self.NUM_FAILURES_TO_DISPLAY])
self._addToLog('stderr', '\nFailures in first run: {}'.format((list(first_run_binary_failures) + list(first_run_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]))
self._addToLog('stderr', '\nFailures in second run: {}'.format((list(second_run_binary_failures) + list(second_run_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]))
@@ -1998,23 +2000,18 @@
self._addToLog('stderr', '\nFlaky Tests: {}'.format(flaky_failures_string))
self._addToLog('stderr', '\nFailures on clean tree: {}'.format(clean_tree_failures_string))
+ if (not first_run_stress_failures) and (not first_run_binary_failures) and (not second_run_stress_failures) and (not second_run_binary_failures):
+ # If we've made it here, then jsc-tests and re-run-jsc-tests failed, which means
+ # there should have been some test failures. Otherwise there is some unexpected issue.
+ clean_tree_run_status = self.getProperty('clean_tree_run_status', FAILURE)
+ if clean_tree_run_status == SUCCESS:
+ return self.report_failure(set(), set())
+ # TODO: email EWS admins
+ return self.retry_build('Unexpected infrastructure issue, retrying build')
+
if new_stress_failures or new_binary_failures:
- self._addToLog('stderr', '\nNew binary failures: {}.\nNew stress test failures: {}\n'.format(new_binary_failures_to_display, new_stress_failures_to_display))
- self.finished(FAILURE)
- self.build.results = FAILURE
- message = ''
- if new_binary_failures:
- pluralSuffix = 's' if len(new_binary_failures) > 1 else ''
- message = 'Found {} new JSC binary failure{}: {}'.format(len(new_binary_failures), pluralSuffix, new_binary_failures_to_display)
- if new_stress_failures:
- if message:
- message += ', '
- pluralSuffix = 's' if len(new_stress_failures) > 1 else ''
- message += 'Found {} new JSC stress test failure{}: {}'.format(len(new_stress_failures), pluralSuffix, new_stress_failures_to_display)
- if len(new_stress_failures) > self.NUM_FAILURES_TO_DISPLAY:
- message += ' ...'
- self.descriptionDone = message
- self.build.buildFinished([message], FAILURE)
+ self._addToLog('stderr', '\nNew binary failures: {}.\nNew stress test failures: {}\n'.format(self.new_binary_failures_to_display, self.new_stress_failures_to_display))
+ return self.report_failure(new_binary_failures, new_stress_failures)
else:
self._addToLog('stderr', '\nNo new failures\n')
self.finished(SUCCESS)
@@ -2035,6 +2032,33 @@
self.build.buildFinished([message], SUCCESS)
return defer.succeed(None)
+ def retry_build(self, message):
+ self.descriptionDone = message
+ self.finished(RETRY)
+ self.build.buildFinished([message], RETRY)
+ return defer.succeed(None)
+
+ def report_failure(self, new_binary_failures, new_stress_failures):
+ message = ''
+ if (not new_binary_failures) and (not new_stress_failures):
+ message = 'Found unexpected failure with patch'
+ if new_binary_failures:
+ pluralSuffix = 's' if len(new_binary_failures) > 1 else ''
+ message = 'Found {} new JSC binary failure{}: {}'.format(len(new_binary_failures), pluralSuffix, self.new_binary_failures_to_display)
+ if new_stress_failures:
+ if message:
+ message += ', '
+ pluralSuffix = 's' if len(new_stress_failures) > 1 else ''
+ message += 'Found {} new JSC stress test failure{}: {}'.format(len(new_stress_failures), pluralSuffix, self.new_stress_failures_to_display)
+ if len(new_stress_failures) > self.NUM_FAILURES_TO_DISPLAY:
+ message += ' ...'
+
+ self.finished(FAILURE)
+ self.build.results = FAILURE
+ self.descriptionDone = message
+ self.build.buildFinished([message], FAILURE)
+ return defer.succeed(None)
+
@defer.inlineCallbacks
def _addToLog(self, logName, message):
try:
Modified: trunk/Tools/CISupport/ews-build/steps_unittest.py (281760 => 281761)
--- trunk/Tools/CISupport/ews-build/steps_unittest.py 2021-08-30 15:56:19 UTC (rev 281760)
+++ trunk/Tools/CISupport/ews-build/steps_unittest.py 2021-08-30 16:02:14 UTC (rev 281761)
@@ -1653,7 +1653,25 @@
self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests')
return self.runStep()
+ def test_unexpected_infra_issue(self):
+ self.configureStep()
+ self.setProperty('jsc_stress_test_failures', [])
+ self.setProperty('jsc_rerun_stress_test_failures', [])
+ self.setProperty('jsc_clean_tree_stress_test_failures', [])
+ self.setProperty('clean_tree_run_status', FAILURE)
+ self.expectOutcome(result=RETRY, state_string='Unexpected infrastructure issue, retrying build (retry)')
+ return self.runStep()
+ def test_patch_breaking_jsc_test_suite(self):
+ self.configureStep()
+ self.setProperty('jsc_stress_test_failures', [])
+ self.setProperty('jsc_rerun_stress_test_failures', [])
+ self.setProperty('jsc_clean_tree_stress_test_failures', [])
+ self.setProperty('clean_tree_run_status', SUCCESS)
+ self.expectOutcome(result=FAILURE, state_string='Found unexpected failure with patch (failure)')
+ return self.runStep()
+
+
class TestRunWebKitTests(BuildStepMixinAdditions, unittest.TestCase):
def setUp(self):
self.longMessage = True
Modified: trunk/Tools/ChangeLog (281760 => 281761)
--- trunk/Tools/ChangeLog 2021-08-30 15:56:19 UTC (rev 281760)
+++ trunk/Tools/ChangeLog 2021-08-30 16:02:14 UTC (rev 281761)
@@ -1,3 +1,17 @@
+2021-08-30 Aakash Jain <aakash_j...@apple.com>
+
+ ews might mark build as successful if JSC tests fail to run
+ https://bugs.webkit.org/show_bug.cgi?id=229634
+
+ Reviewed by Jonathan Bedard.
+
+ * CISupport/ews-build/steps.py:
+ (RunJSCTestsWithoutPatch.evaluateCommand):
+ (AnalyzeJSCTestsResults.start):
+ (AnalyzeJSCTestsResults.retry_build): Method to retry build.
+ (AnalyzeJSCTestsResults.report_failure):
+ * CISupport/ews-build/steps_unittest.py: Added unit-tests.
+
2021-08-30 Lauro Moura <lmo...@igalia.com>
[WPE][webkitpy] Sanitize browser name testcase environment
