Modified: trunk/Tools/BuildSlaveSupport/ews-build/steps.py (252429 => 252430)
--- trunk/Tools/BuildSlaveSupport/ews-build/steps.py 2019-11-13 21:47:09 UTC (rev 252429)
+++ trunk/Tools/BuildSlaveSupport/ews-build/steps.py 2019-11-13 21:51:48 UTC (rev 252430)
@@ -946,11 +946,18 @@
jsonFileName = 'jsc_results.json'
logfiles = {'json': jsonFileName}
command = ['perl', 'Tools/Scripts/run-_javascript_core-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(jsonFileName), WithProperties('--%(configuration)s')]
+ prefix = 'jsc_'
+ NUM_FAILURES_TO_DISPLAY_IN_STATUS = 5
def __init__(self, **kwargs):
shell.Test.__init__(self, logEnviron=False, **kwargs)
+ self.binaryFailures = []
+ self.stressTestFailures = []
def start(self):
+ self.log_observer_json = logobserver.BufferLogObserver()
+ self.addLogObserver('json', self.log_observer_json)
+
# add remotes configuration file path to the command line if needed
remotesfile = self.getProperty('remotes', False)
if remotesfile:
@@ -970,9 +977,65 @@
self.build.addStepsAfterCurrentStep([ReRunJavaScriptCoreTests()])
return rc
+ def commandComplete(self, cmd):
+ shell.Test.commandComplete(self, cmd)
+ logLines = self.log_observer_json.getStdout()
+ json_text = ''.join([line for line in logLines.splitlines()])
+ try:
+ jsc_results = json.loads(json_text)
+ except Exception as ex:
+ self._addToLog('stderr', 'ERROR: unable to parse data, exception: {}'.format(ex))
+ return
+ if jsc_results.get('allMasmTestsPassed') == False:
+ self.binaryFailures.append('testmasm')
+ if jsc_results.get('allAirTestsPassed') == False:
+ self.binaryFailures.append('testair')
+ if jsc_results.get('allB3TestsPassed') == False:
+ self.binaryFailures.append('testb3')
+ if jsc_results.get('allDFGTestsPassed') == False:
+ self.binaryFailures.append('testdfg')
+ if jsc_results.get('allApiTestsPassed') == False:
+ self.binaryFailures.append('testapi')
+
+ self.stressTestFailures = jsc_results.get('stressTestFailures')
+ if self.stressTestFailures:
+ self.setProperty(self.prefix + 'stress_test_failures', self.stressTestFailures)
+ if self.binaryFailures:
+ self.setProperty(self.prefix + 'binary_failures', self.binaryFailures)
+
+ def getResultSummary(self):
+ if self.results != SUCCESS and (self.stressTestFailures or self.binaryFailures):
+ status = ''
+ if self.stressTestFailures:
+ num_failures = len(self.stressTestFailures)
+ pluralSuffix = 's' if num_failures > 1 else ''
+ failures_to_display = self.stressTestFailures[:self.NUM_FAILURES_TO_DISPLAY_IN_STATUS]
+ status = 'Found {} jsc stress test failure{}: '.format(num_failures, pluralSuffix) + ', '.join(failures_to_display)
+ if num_failures > self.NUM_FAILURES_TO_DISPLAY_IN_STATUS:
+ status += ' ...'
+ if self.binaryFailures:
+ if status:
+ status += ', '
+ pluralSuffix = 's' if len(self.binaryFailures) > 1 else ''
+ status += 'JSC test binary failure{}: {}'.format(pluralSuffix, ', '.join(self.binaryFailures))
+
+ return {u'step': unicode(status)}
+
+ return shell.Test.getResultSummary(self)
+
+ @defer.inlineCallbacks
+ def _addToLog(self, logName, message):
+ try:
+ log = self.getLog(logName)
+ except KeyError:
+ log = yield self.addLog(logName)
+ log.addStdout(message)
+
+
class ReRunJavaScriptCoreTests(RunJavaScriptCoreTests):
name = 'jscore-test-rerun'
+ prefix = 'jsc_rerun_'
def evaluateCommand(self, cmd):
rc = shell.Test.evaluateCommand(self, cmd)
@@ -989,7 +1052,7 @@
class RunJSCTestsWithoutPatch(RunJavaScriptCoreTests):
name = 'jscore-test-without-patch'
- jsonFileName = 'jsc_results.json'
+ prefix = 'jsc_clean_tree_'
def evaluateCommand(self, cmd):
return shell.Test.evaluateCommand(self, cmd)
Modified: trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py (252429 => 252430)
--- trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py 2019-11-13 21:47:09 UTC (rev 252429)
+++ trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py 2019-11-13 21:51:48 UTC (rev 252430)
@@ -999,6 +999,11 @@
def setUp(self):
self.longMessage = True
self.jsonFileName = 'jsc_results.json'
+ self.jsc_masm_failure = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":false,"allB3TestsPassed":true,"allAirTestsPassed":true,"stressTestFailures":[],"allApiTestsPassed":true}\n'''
+ self.jsc_b3_and_stress_test_failure = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":true,"allB3TestsPassed":false,"allAirTestsPassed":true,"allApiTestsPassed":true,"stressTestFailures":["stress/weakset-gc.js"]}\n'''
+ self.jsc_dfg_air_and_stress_test_failure = '''{"allDFGTestsPassed":false,"allMasmTestsPassed":true,"allB3TestsPassed":true,"allAirTestsPassed":false,"allApiTestsPassed":true,"stressTestFailures":["stress/weakset-gc.js"]}\n'''
+ self.jsc_single_stress_test_failure = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":true,"allB3TestsPassed":true,"allAirTestsPassed":true,"stressTestFailures":["stress/switch-on-char-llint-rope.js.dfg-eager"],"allApiTestsPassed":true}\n'''
+ self.jsc_multiple_stress_test_failures = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":true,"allB3TestsPassed":true,"allAirTestsPassed":true,"stressTestFailures":["stress/switch-on-char-llint-rope.js.dfg-eager","stress/switch-on-char-llint-rope.js.dfg-eager-no-cjit-validate","stress/switch-on-char-llint-rope.js.eager-jettison-no-cjit","stress/switch-on-char-llint-rope.js.ftl-eager","stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit","stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit-b3o1","stress/switch-on-char-llint-rope.js.ftl-no-cjit-b3o0","stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-inline-validate","stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-put-stack-validate","stress/switch-on-char-llint-rope.js.ftl-no-cjit-small-pool","stress/switch-on-char-llint-rope.js.ftl-no-cjit-validate-sampling-prof
iler","stress/switch-on-char-llint-rope.js.no-cjit-collect-continuously","stress/switch-on-char-llint-rope.js.no-cjit-validate-phases","stress/switch-on-char-llint-rope.js.no-ftl"],"allApiTestsPassed":true}\n'''
return self.setUpBuildStep()
def tearDown(self):
@@ -1006,6 +1011,7 @@
def configureStep(self, platform=None, fullPlatform=None, configuration=None):
self.setupStep(RunJavaScriptCoreTests())
+ self.prefix = RunJavaScriptCoreTests.prefix
if platform:
self.setProperty('platform', platform)
if fullPlatform:
@@ -1054,10 +1060,96 @@
self.expectOutcome(result=FAILURE, state_string='jscore-tests (failure)')
return self.runStep()
+ def test_single_stress_test_failure(self):
+ self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='debug')
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logEnviron=False,
+ logfiles={'json': self.jsonFileName},
+ command=['perl', 'Tools/Scripts/run-_javascript_core-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'],
+ )
+ + 2
+ + ExpectShell.log('json', stdout=self.jsc_single_stress_test_failure),
+ )
+ self.expectOutcome(result=FAILURE, state_string='Found 1 jsc stress test failure: stress/switch-on-char-llint-rope.js.dfg-eager')
+ rc = self.runStep()
+ self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ['stress/switch-on-char-llint-rope.js.dfg-eager'])
+ self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), None)
+ return rc
+ def test_lot_of_stress_test_failure(self):
+ self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='debug')
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logEnviron=False,
+ logfiles={'json': self.jsonFileName},
+ command=['perl', 'Tools/Scripts/run-_javascript_core-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'],
+ )
+ + 2
+ + ExpectShell.log('json', stdout=self.jsc_multiple_stress_test_failures),
+ )
+ self.expectOutcome(result=FAILURE, state_string='Found 14 jsc stress test failures: stress/switch-on-char-llint-rope.js.dfg-eager, stress/switch-on-char-llint-rope.js.dfg-eager-no-cjit-validate, stress/switch-on-char-llint-rope.js.eager-jettison-no-cjit, stress/switch-on-char-llint-rope.js.ftl-eager, stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit ...')
+ rc = self.runStep()
+ self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ["stress/switch-on-char-llint-rope.js.dfg-eager", "stress/switch-on-char-llint-rope.js.dfg-eager-no-cjit-validate", "stress/switch-on-char-llint-rope.js.eager-jettison-no-cjit", "stress/switch-on-char-llint-rope.js.ftl-eager", "stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit", "stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit-b3o1", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-b3o0", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-inline-validate", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-put-stack-validate", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-small-pool", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-validate-sampling-profiler", "stress/switch-on-char-llint-rope.js.no-cjit-collect-continuously", "stress/switch-on-char-llint-rope.js.n
o-cjit-validate-phases", "stress/switch-on-char-llint-rope.js.no-ftl"])
+ self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), None)
+ return rc
+
+ def test_masm_failure(self):
+ self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='debug')
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logEnviron=False,
+ logfiles={'json': self.jsonFileName},
+ command=['perl', 'Tools/Scripts/run-_javascript_core-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'],
+ )
+ + 2
+ + ExpectShell.log('json', stdout=self.jsc_masm_failure),
+ )
+ self.expectOutcome(result=FAILURE, state_string='JSC test binary failure: testmasm')
+ rc = self.runStep()
+ self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), None)
+ self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), ['testmasm'])
+ return rc
+
+ def test_b3_and_stress_test_failure(self):
+ self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='release')
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logEnviron=False,
+ logfiles={'json': self.jsonFileName},
+ command=['perl', 'Tools/Scripts/run-_javascript_core-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release'],
+ )
+ + 2
+ + ExpectShell.log('json', stdout=self.jsc_b3_and_stress_test_failure),
+ )
+ self.expectOutcome(result=FAILURE, state_string='Found 1 jsc stress test failure: stress/weakset-gc.js, JSC test binary failure: testb3')
+ rc = self.runStep()
+ self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ['stress/weakset-gc.js'])
+ self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), ['testb3'])
+ return rc
+
+ def test_dfg_air_and_stress_test_failure(self):
+ self.configureStep(platform='jsc-only', fullPlatform='jsc-only', configuration='release')
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ logEnviron=False,
+ logfiles={'json': self.jsonFileName},
+ command=['perl', 'Tools/Scripts/run-_javascript_core-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release', '--jsc-only'],
+ )
+ + 2
+ + ExpectShell.log('json', stdout=self.jsc_dfg_air_and_stress_test_failure),
+ )
+ self.expectOutcome(result=FAILURE, state_string='Found 1 jsc stress test failure: stress/weakset-gc.js, JSC test binary failures: testair, testdfg')
+ rc = self.runStep()
+ self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ['stress/weakset-gc.js'])
+ self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), ['testair', 'testdfg'])
+ return rc
+
+
class TestReRunJavaScriptCoreTests(TestRunJavaScriptCoreTests):
def configureStep(self, platform=None, fullPlatform=None, configuration=None):
self.setupStep(ReRunJavaScriptCoreTests())
+ self.prefix = ReRunJavaScriptCoreTests.prefix
if platform:
self.setProperty('platform', platform)
if fullPlatform:
