Diff
Modified: trunk/Tools/BuildSlaveSupport/ews-build/send_email.py (264794 => 264795)
--- trunk/Tools/BuildSlaveSupport/ews-build/send_email.py 2020-07-23 22:17:50 UTC (rev 264794)
+++ trunk/Tools/BuildSlaveSupport/ews-build/send_email.py 2020-07-23 22:20:23 UTC (rev 264795)
@@ -28,6 +28,13 @@
def send_email(to_emails, subject, text):
+ if not to_emails:
+ print('Skipping email since no recipient is specified')
+ return
+ if not subject or not text:
+ print('Skipping email since no subject or text is specified')
+ return
+
email_data = """From: {}\nTo: {}\nSubject: {}\n\n{}""".format(FROM_EMAIL, ', '.join(to_emails), subject, text)
server = smtplib.SMTP(SERVER)
Modified: trunk/Tools/BuildSlaveSupport/ews-build/steps.py (264794 => 264795)
--- trunk/Tools/BuildSlaveSupport/ews-build/steps.py 2020-07-23 22:17:50 UTC (rev 264794)
+++ trunk/Tools/BuildSlaveSupport/ews-build/steps.py 2020-07-23 22:20:23 UTC (rev 264795)
@@ -29,6 +29,7 @@
from twisted.internet import defer
from layout_test_failures import LayoutTestFailures
+from send_email import send_email_to_bot_watchers
import json
import re
@@ -1866,6 +1867,8 @@
pluralSuffix = 's' if len(flaky_failures) > 1 else ''
message = 'Found flaky test{}: {}'.format(pluralSuffix, flaky_failures_string)
self.setProperty('build_summary', message)
+ for flaky_failure in flaky_failures:
+ self.send_email_for_flaky_failure(flaky_failure)
else:
self.setProperty('patchFailedTests', True)
self.build.addStepsAfterCurrentStep([ArchiveTestResults(),
@@ -1893,7 +1896,19 @@
self._addToLog(self.test_failures_log_name, '\n'.join(second_results.failing_tests))
self._parseRunWebKitTestsOutput(logText)
+ def send_email_for_flaky_failure(self, test_name):
+ try:
+ builder_name = self.getProperty('buildername', '')
+ build_url = '{}#/builders/{}/builds/{}'.format(self.master.config.buildbotURL, self.build._builderid, self.build.number)
+ email_subject = 'Flaky test: {}'.format(test_name)
+ email_text = 'Test {} flaked in {}\n\nBuilder: {}'.format(test_name, build_url, builder_name)
+ send_email_to_bot_watchers(email_subject, email_text)
+ except Exception as e:
+ # Catching all exceptions here to ensure that failure to send email doesn't impact the build
+ print('Error in sending email for flaky failures: {}'.format(e))
+
+
class RunWebKitTestsWithoutPatch(RunWebKitTests):
name = 'run-layout-tests-without-patch'
@@ -1952,6 +1967,9 @@
message = 'Found {} pre-existing test failure{}: {}'.format(len(clean_tree_failures), pluralSuffix, clean_tree_failures_string)
if len(clean_tree_failures) > self.NUM_FAILURES_TO_DISPLAY:
message += ' ...'
+ for clean_tree_failure in clean_tree_failures:
+ self.send_email_for_pre_existing_failure(clean_tree_failure)
+
if flaky_failures:
flaky_failures_string = ', '.join(sorted(flaky_failures)[:self.NUM_FAILURES_TO_DISPLAY])
pluralSuffix = 's' if len(flaky_failures) > 1 else ''
@@ -1958,6 +1976,9 @@
message += ' Found flaky test{}: {}'.format(pluralSuffix, flaky_failures_string)
if len(flaky_failures) > self.NUM_FAILURES_TO_DISPLAY:
message += ' ...'
+ for flaky_failure in flaky_failures:
+ self.send_email_for_flaky_failure(flaky_failure)
+
self.setProperty('build_summary', message)
return defer.succeed(None)
@@ -1971,6 +1992,28 @@
def _results_failed_different_tests(self, first_results_failing_tests, second_results_failing_tests):
return first_results_failing_tests != second_results_failing_tests
+ def send_email_for_flaky_failure(self, test_name):
+ try:
+ builder_name = self.getProperty('buildername', '')
+ build_url = '{}#/builders/{}/builds/{}'.format(self.master.config.buildbotURL, self.build._builderid, self.build.number)
+
+ email_subject = 'Flaky test: {}'.format(test_name)
+ email_text = 'Test {} flaked in {}\n\nBuilder: {}'.format(test_name, build_url, builder_name)
+ rc = send_email_to_bot_watchers(email_subject, email_text)
+ except Exception as e:
+ print('Error in sending email for flaky failure: {}'.format(e))
+
+ def send_email_for_pre_existing_failure(self, test_name):
+ try:
+ builder_name = self.getProperty('buildername', '')
+ build_url = '{}#/builders/{}/builds/{}'.format(self.master.config.buildbotURL, self.build._builderid, self.build.number)
+
+ email_subject = 'Pre-existing test failure: {}'.format(test_name)
+ email_text = 'Test {} failed on clean tree run in {}.\nBuilder: {}'.format(test_name, build_url, builder_name)
+ rc = send_email_to_bot_watchers(email_subject, email_text)
+ except Exception as e:
+ print('Error in sending email for pre-existing failure: {}'.format(e))
+
def _report_flaky_tests(self, flaky_tests):
#TODO: implement this
pass
Modified: trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py (264794 => 264795)
--- trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py 2020-07-23 22:17:50 UTC (rev 264794)
+++ trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py 2020-07-23 22:20:23 UTC (rev 264795)
@@ -52,6 +52,9 @@
Trigger, TransferToS3, UnApplyPatchIfRequired, UpdateWorkingDirectory, UploadBuiltProduct,
UploadTestResults, ValidateCommiterAndReviewer, ValidatePatch)
+import send_email
+send_email.BOT_WATCHERS_EMAILS = []
+
# Workaround for https://github.com/buildbot/buildbot/issues/4669
from buildbot.test.fake.fakebuild import FakeBuild
FakeBuild.addStepsAfterCurrentStep = lambda FakeBuild, step_factories: None
@@ -1737,8 +1740,27 @@
self.setupStep(ReRunWebKitTests())
self.property_exceed_failure_limit = 'second_results_exceed_failure_limit'
self.property_failures = 'second_run_failures'
+ ReRunWebKitTests.send_email_for_flaky_failure = lambda subject, test: None
+ def test_flaky_failures_in_first_run(self):
+ self.configureStep()
+ self.setProperty('fullPlatform', 'ios-simulator')
+ self.setProperty('configuration', 'release')
+ self.setProperty('first_run_failures', ['test1', 'test2'])
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logfiles={'json': self.jsonFileName},
+ logEnviron=False,
+ command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'],
+ )
+ + 0,
+ )
+ self.expectOutcome(result=SUCCESS, state_string='Passed layout tests')
+ rc = self.runStep()
+ self.assertEqual(self.getProperty('build_summary'), 'Found flaky tests: test1, test2')
+ return rc
+
class TestRunWebKitTestsWithoutPatch(BuildStepMixinAdditions, unittest.TestCase):
def setUp(self):
self.longMessage = True
@@ -1867,6 +1889,8 @@
return self.tearDownBuildStep()
def configureStep(self):
+ AnalyzeLayoutTestsResults.send_email_for_flaky_failure = lambda subject, test: None
+ AnalyzeLayoutTestsResults.send_email_for_pre_existing_failure = lambda subject, test: None
self.setupStep(AnalyzeLayoutTestsResults())
self.setProperty('first_results_exceed_failure_limit', False)
self.setProperty('second_results_exceed_failure_limit', False)
Modified: trunk/Tools/ChangeLog (264794 => 264795)
--- trunk/Tools/ChangeLog 2020-07-23 22:17:50 UTC (rev 264794)
+++ trunk/Tools/ChangeLog 2020-07-23 22:20:23 UTC (rev 264795)
@@ -1,3 +1,19 @@
+2020-07-23 Aakash Jain <aakash_j...@apple.com>
+
+ [ews] Notify bot watchers about pre-existing and flaky test failures found on ews bots
+ https://bugs.webkit.org/show_bug.cgi?id=207475
+
+ Reviewed by Jonathan Bedard.
+
+ * BuildSlaveSupport/ews-build/steps.py:
+ (ReRunWebKitTests.evaluateCommand):
+ (ReRunWebKitTests.send_email_for_flaky_failure): Method to send email for flaky failure.
+ (AnalyzeLayoutTestsResults.report_pre_existing_failures):
+ (AnalyzeLayoutTestsResults.send_email_for_flaky_failure): Method to send email for flaky failure.
+ (AnalyzeLayoutTestsResults.send_email_for_pre_existing_failure): Method to send email for pre-existing failure.
+ * BuildSlaveSupport/ews-build/steps.py: Updated unit-tests accordingly.
+ * BuildSlaveSupport/ews-build/send_email.py: Added error handling.
+
2020-07-23 Sihui Liu <sihui_...@appe.com>
Allow IndexedDB in third-party frames
