Modified: trunk/Tools/BuildSlaveSupport/ews-build/steps.py (234279 => 234280)
--- trunk/Tools/BuildSlaveSupport/ews-build/steps.py 2018-07-26 22:16:28 UTC (rev 234279)
+++ trunk/Tools/BuildSlaveSupport/ews-build/steps.py 2018-07-26 22:51:39 UTC (rev 234280)
@@ -20,12 +20,14 @@
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-from buildbot.process import buildstep, properties
+from buildbot.process import buildstep, logobserver, properties
from buildbot.process.results import Results, SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION, RETRY
from buildbot.steps import shell, transfer
from buildbot.steps.source import svn
from twisted.internet import defer
+import re
+
EWS_URL = 'http://ews-build.webkit-uat.org/'
WithProperties = properties.WithProperties
@@ -89,6 +91,44 @@
command = ['Tools/Scripts/check-webkit-style']
+class TestWithFailureCount(shell.Test):
+ failedTestsFormatString = "%d test%s failed"
+ failedTestCount = 0
+
+ def start(self):
+ self.log_observer = logobserver.BufferLogObserver(wantStderr=True)
+ self.addLogObserver('stdio', self.log_observer)
+ return shell.Test.start(self)
+
+ def countFailures(self, cmd):
+ raise NotImplementedError
+
+ def commandComplete(self, cmd):
+ shell.Test.commandComplete(self, cmd)
+ self.failedTestCount = self.countFailures(cmd)
+ self.failedTestPluralSuffix = "" if self.failedTestCount == 1 else "s"
+
+ def evaluateCommand(self, cmd):
+ if self.failedTestCount:
+ return FAILURE
+
+ if cmd.rc != 0:
+ return FAILURE
+
+ return SUCCESS
+
+ def getResultSummary(self):
+ status = self.name
+
+ if self.results != SUCCESS and self.failedTestCount:
+ status = self.failedTestsFormatString % (self.failedTestCount, self.failedTestPluralSuffix)
+
+ if self.results != SUCCESS:
+ status += u' ({})'.format(Results[self.results])
+
+ return {u'step': status}
+
+
class RunBindingsTests(shell.ShellCommand):
name = 'bindings-tests'
description = ['bindings-tests running']
@@ -330,3 +370,23 @@
descriptionDone = ['extracted built product']
haltOnFailure = True
flunkOnFailure = True
+
+
+class RunAPITests(TestWithFailureCount):
+ name = 'run-api-tests'
+ description = ['api tests running']
+ descriptionDone = ['api-tests']
+ command = ['python', 'Tools/Scripts/run-api-tests', '--no-build', WithProperties('--%(configuration)s'), '--verbose']
+ failedTestsFormatString = '%d api test%s failed or timed out'
+
+ def start(self):
+ appendCustomBuildFlags(self, self.getProperty('platform'), self.getProperty('fullPlatform'))
+ return TestWithFailureCount.start(self)
+
+ def countFailures(self, cmd):
+ log_text = self.log_observer.getStdout() + self.log_observer.getStderr()
+
+ match = re.search(r'Ran (?P<ran>\d+) tests of (?P<total>\d+) with (?P<passed>\d+) successful', log_text)
+ if not match:
+ return 0
+ return int(match.group('ran')) - int(match.group('passed'))
Modified: trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py (234279 => 234280)
--- trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py 2018-07-26 22:16:28 UTC (rev 234279)
+++ trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py 2018-07-26 22:51:39 UTC (rev 234280)
@@ -858,5 +858,203 @@
return self.runStep()
+class TestRunAPITests(BuildStepMixinAdditions, unittest.TestCase):
+ def setUp(self):
+ self.longMessage = True
+ return self.setUpBuildStep()
+
+ def tearDown(self):
+ return self.tearDownBuildStep()
+
+ def test_success_mac(self):
+ self.setupStep(RunAPITests())
+ self.setProperty('fullPlatform', 'mac-mojave')
+ self.setProperty('platform', 'mac')
+ self.setProperty('configuration', 'release')
+
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--release', '--verbose'],
+ )
+ + ExpectShell.log('stdio', stdout='''...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+Ran 1888 tests of 1888 with 1888 successful
+------------------------------
+All tests successfully passed!
+''')
+ + 0,
+ )
+ self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
+ return self.runStep()
+
+ def test_success_ios_simulator(self):
+ self.setupStep(RunAPITests())
+ self.setProperty('fullPlatform', 'ios-simulator-11')
+ self.setProperty('platform', 'ios')
+ self.setProperty('configuration', 'debug')
+
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose', '--ios-simulator'],
+ )
+ + ExpectShell.log('stdio', stdout='''...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+Ran 1888 tests of 1888 with 1888 successful
+------------------------------
+All tests successfully passed!
+''')
+ + 0,
+ )
+ self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
+ return self.runStep()
+
+ def test_one_failure(self):
+ self.setupStep(RunAPITests())
+ self.setProperty('fullPlatform', 'mac-mojave')
+ self.setProperty('platform', 'mac')
+ self.setProperty('configuration', 'debug')
+
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
+ )
+ + ExpectShell.log('stdio', stdout='''
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1888 tests of 1888 with 1887 successful
+------------------------------
+Test suite failed
+
+Crashed
+
+ TestWTF.WTF.StringConcatenate_Unsigned
+ **FAIL** WTF.StringConcatenate_Unsigned
+
+ Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
+ Value of: makeString('hello ', static_cast<unsigned short>(42) , ' world')
+ Actual: hello 42 world
+ Expected: 'hello * world'
+ Which is: 74B00C9C
+
+Testing completed, Exit status: 3
+''')
+ + 1,
+ )
+ self.expectOutcome(result=FAILURE, state_string='1 api test failed or timed out (failure)')
+ return self.runStep()
+
+ def test_multiple_failures_and_timeouts(self):
+ self.setupStep(RunAPITests())
+ self.setProperty('fullPlatform', 'mac-mojave')
+ self.setProperty('platform', 'mac')
+ self.setProperty('configuration', 'debug')
+
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
+ )
+ + ExpectShell.log('stdio', stdout='''...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1888 tests of 1888 with 1884 successful
+------------------------------
+Test suite failed
+
+Failed
+
+ TestWTF.WTF.StringConcatenate_Unsigned
+ **FAIL** WTF.StringConcatenate_Unsigned
+
+ Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
+ Value of: makeString('hello ', static_cast<unsigned short>(42) , ' world')
+ Actual: hello 42 world
+ Expected: 'hello * world'
+ Which is: 74B00C9C
+
+ TestWTF.WTF_Expected.Unexpected
+ **FAIL** WTF_Expected.Unexpected
+
+ Tools\TestWebKitAPI\Tests\WTF\Expected.cpp:96
+ Value of: s1
+ Actual: oops
+ Expected: s0
+ Which is: oops
+
+Timeout
+
+ TestWTF.WTF_PoisonedUniquePtrForTriviallyDestructibleArrays.Assignment
+ TestWTF.WTF_Lock.ContendedShortSection
+
+Testing completed, Exit status: 3
+''')
+ + 4,
+ )
+ self.expectOutcome(result=FAILURE, state_string='4 api tests failed or timed out (failure)')
+ return self.runStep()
+
+ def test_unexpecte_failure(self):
+ self.setupStep(RunAPITests())
+ self.setProperty('fullPlatform', 'mac-mojave')
+ self.setProperty('platform', 'mac')
+ self.setProperty('configuration', 'debug')
+
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
+ )
+ + ExpectShell.log('stdio', stdout='Unexpected failure. Failed to run api tests.')
+ + 2,
+ )
+ self.expectOutcome(result=FAILURE, state_string='run-api-tests (failure)')
+ return self.runStep()
+
+ def test_no_failures_or_timeouts_with_disabled(self):
+ self.setupStep(RunAPITests())
+ self.setProperty('fullPlatform', 'mac-mojave')
+ self.setProperty('platform', 'mac')
+ self.setProperty('configuration', 'debug')
+
+ self.expectRemoteCommands(
+ ExpectShell(workdir='wkdir',
+ command=['python', 'Tools/Scripts/run-api-tests', '--no-build', '--debug', '--verbose'],
+ )
+ + ExpectShell.log('stdio', stdout='''...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1881 tests of 1888 with 1881 successful
+------------------------------
+All tests successfully passed!
+''')
+ + 0,
+ )
+ self.expectOutcome(result=SUCCESS, state_string='run-api-tests')
+ return self.runStep()
+
+
if __name__ == '__main__':
unittest.main()