Title: [231039] trunk/Tools
- Revision
- 231039
- Author
- [email protected]
- Date
- 2018-04-25 19:10:59 -0700 (Wed, 25 Apr 2018)
Log Message
REGRESSION (r230998): Bot watcher's dashboard doesn't display number of API test failures
https://bugs.webkit.org/show_bug.cgi?id=184982
Rubber-stamped by Aakash Jain.
Correctly count the number of API test failures.
* BuildSlaveSupport/build.webkit.org-config/steps.py:
(RunUnitTests.countFailures): Count the number of test failures for the new run-api-tests.
* BuildSlaveSupport/build.webkit.org-config/steps_unittest.py:
* Scripts/webkitpy/api_tests/manager.py:
(Manager.run): Disabled tests are not run.
Modified Paths
Diff
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps.py (231038 => 231039)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps.py 2018-04-26 02:04:48 UTC (rev 231038)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps.py 2018-04-26 02:10:59 UTC (rev 231039)
@@ -496,17 +496,12 @@
log_text = cmd.logs['stdio'].getText()
count = 0
- split = re.split(r'\sTests that timed out:\s', log_text)
- if len(split) > 1:
- count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
+ match = re.search(r'Ran (?P<ran>\d+) tests of (?P<total>\d+) with (?P<passed>\d+) successful', log_text)
+ if not match:
+ return -1
+ return int(match.group('total')) - int(match.group('passed'))
- split = re.split(r'\sTests that failed:\s', split[0])
- if len(split) > 1:
- count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
- return count
-
-
class RunPythonTests(TestWithFailureCount):
name = "webkitpy-test"
description = ["python-tests running"]
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps_unittest.py (231038 => 231039)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps_unittest.py 2018-04-26 02:04:48 UTC (rev 231038)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps_unittest.py 2018-04-26 02:10:59 UTC (rev 231039)
@@ -228,111 +228,162 @@
self.assertEqual(expected_text, actual_text)
def test_no_failures_or_timeouts(self):
- self.assertFailures(0, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
-
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[ PASSED ] 1 test.
+ self.assertFailures(0, """...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1888 tests of 1888 with 1888 successful
+------------------------------
+All tests successfully passed!
""")
def test_one_failure(self):
- self.assertFailures(1, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+ self.assertFailures(1, """...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1888 tests of 1888 with 1887 successful
+------------------------------
+Test suite failed
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[ PASSED ] 1 test.
-Tests that failed:
- WebKit2.WebKit2.CanHandleRequest
+Crashed
+
+ TestWTF.WTF.StringConcatenate_Unsigned
+ **FAIL** WTF.StringConcatenate_Unsigned
+
+ C:\\cygwin\\home\\buildbot\\slave\\win-release\\build\\Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
+ Value of: makeString("hello ", static_cast<unsigned short>(42) , " world")
+ Actual: hello 42 world
+ Expected: "hello * world"
+ Which is: 74B00C9C
+
+Testing completed, Exit status: 3
""")
def test_multiple_failures(self):
- self.assertFailures(4, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+ self.assertFailures(2, """...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1888 tests of 1888 with 1886 successful
+------------------------------
+Test suite failed
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[ PASSED ] 1 test.
-Tests that failed:
- WebKit2.WebKit2.CanHandleRequest
- WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
- WebKit2.WebKit2.HitTestResultNodeHandle
- WebKit2.WebKit2.InjectedBundleBasic
+Crashed
+
+ TestWTF.WTF.StringConcatenate_Unsigned
+ **FAIL** WTF.StringConcatenate_Unsigned
+
+ C:\\cygwin\\home\\buildbot\\slave\\win-release\\build\\Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
+ Value of: makeString("hello ", static_cast<unsigned short>(42) , " world")
+ Actual: hello 42 world
+ Expected: "hello * world"
+ Which is: 74B00C9C
+
+ TestWTF.WTF_Expected.Unexpected
+ **FAIL** WTF_Expected.Unexpected
+
+ C:\cygwin\home\buildbot\slave\win-release\build\Tools\TestWebKitAPI\Tests\WTF\Expected.cpp:96
+ Value of: s1
+ Actual: oops
+ Expected: s0
+ Which is: oops
+
+Testing completed, Exit status: 3
""")
def test_one_timeout(self):
- self.assertFailures(1, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+ self.assertFailures(1, """...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1888 tests of 1888 with 1887 successful
+------------------------------
+Test suite failed
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[ PASSED ] 1 test.
-Tests that timed out:
- WebKit2.WebKit2.CanHandleRequest
+Timeout
+
+ TestWTF.WTF_PoisonedUniquePtrForTriviallyDestructibleArrays.Assignment
+
+Testing completed, Exit status: 3
""")
def test_multiple_timeouts(self):
- self.assertFailures(4, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+ self.assertFailures(2, """...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1888 tests of 1888 with 1886 successful
+------------------------------
+Test suite failed
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[ PASSED ] 1 test.
-Tests that timed out:
- WebKit2.WebKit2.CanHandleRequest
- WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
- WebKit2.WebKit2.HitTestResultNodeHandle
- WebKit2.WebKit2.InjectedBundleBasic
+Timeout
+
+ TestWTF.WTF_PoisonedUniquePtrForTriviallyDestructibleArrays.Assignment
+ TestWTF.WTF_Lock.ContendedShortSection
+
+Testing completed, Exit status: 3
""")
def test_multiple_failures_and_timeouts(self):
- self.assertFailures(8, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[==========] Running 1 test from 1 test case.
-[----------] Global test environment set-up.
-[----------] 1 test from WebViewDestructionWithHostWindow
-[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
-[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
-[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+ self.assertFailures(4, """...
+worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed
+worker/0 TestWTF.WTF_Variant.Ref Passed
+worker/0 TestWTF.WTF_Variant.RefPtr Passed
+worker/0 TestWTF.WTF_Variant.RetainPtr Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed
+worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed
+worker/0 exiting
+Ran 1888 tests of 1888 with 1884 successful
+------------------------------
+Test suite failed
-[----------] Global test environment tear-down
-[==========] 1 test from 1 test case ran. (127 ms total)
-[ PASSED ] 1 test.
-Tests that failed:
- WebKit2.WebKit2.CanHandleRequest
- WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
- WebKit2.WebKit2.HitTestResultNodeHandle
-Tests that timed out:
- WebKit2.WebKit2.InjectedBundleBasic
- WebKit2.WebKit2.LoadCanceledNoServerRedirectCallback
- WebKit2.WebKit2.MouseMoveAfterCrash
- WebKit2.WebKit2.ResponsivenessTimerDoesntFireEarly
- WebKit2.WebKit2.WebArchive
+Crashed
+
+ TestWTF.WTF.StringConcatenate_Unsigned
+ **FAIL** WTF.StringConcatenate_Unsigned
+
+ C:\\cygwin\\home\\buildbot\\slave\\win-release\\build\\Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84
+ Value of: makeString("hello ", static_cast<unsigned short>(42) , " world")
+ Actual: hello 42 world
+ Expected: "hello * world"
+ Which is: 74B00C9C
+
+ TestWTF.WTF_Expected.Unexpected
+ **FAIL** WTF_Expected.Unexpected
+
+ C:\cygwin\home\buildbot\slave\win-release\build\Tools\TestWebKitAPI\Tests\WTF\Expected.cpp:96
+ Value of: s1
+ Actual: oops
+ Expected: s0
+ Which is: oops
+
+Timeout
+
+ TestWTF.WTF_PoisonedUniquePtrForTriviallyDestructibleArrays.Assignment
+ TestWTF.WTF_Lock.ContendedShortSection
+
+Testing completed, Exit status: 3
""")
Modified: trunk/Tools/ChangeLog (231038 => 231039)
--- trunk/Tools/ChangeLog 2018-04-26 02:04:48 UTC (rev 231038)
+++ trunk/Tools/ChangeLog 2018-04-26 02:10:59 UTC (rev 231039)
@@ -1,3 +1,18 @@
+2018-04-25 Jonathan Bedard <[email protected]>
+
+ REGRESSION (r230998): Bot watcher's dashboard doesn't display number of API test failures
+ https://bugs.webkit.org/show_bug.cgi?id=184982
+
+ Rubber-stamped by Aakash Jain.
+
+ Correctly count the number of API test failures.
+
+ * BuildSlaveSupport/build.webkit.org-config/steps.py:
+ (RunUnitTests.countFailures): Count the number of test failures for the new run-api-tests.
+ * BuildSlaveSupport/build.webkit.org-config/steps_unittest.py:
+ * Scripts/webkitpy/api_tests/manager.py:
+ (Manager.run): Disabled tests are not run.
+
2018-04-25 Jiewen Tan <[email protected]>
Remove access to keychain from the WebContent process
Modified: trunk/Tools/Scripts/webkitpy/api_tests/manager.py (231038 => 231039)
--- trunk/Tools/Scripts/webkitpy/api_tests/manager.py 2018-04-26 02:04:48 UTC (rev 231038)
+++ trunk/Tools/Scripts/webkitpy/api_tests/manager.py 2018-04-26 02:10:59 UTC (rev 231039)
@@ -163,10 +163,11 @@
self._stream.writeln('')
successful = runner.result_map_by_status(runner.STATUS_PASSED)
- _log.info('Ran {} tests of {} with {} successful'.format(len(runner.results), len(test_names), len(successful)))
+ disabled = len(runner.result_map_by_status(runner.STATUS_DISABLED))
+ _log.info('Ran {} tests of {} with {} successful'.format(len(runner.results) - disabled, len(test_names), len(successful)))
self._stream.writeln('------------------------------')
- if len(successful) + len(runner.result_map_by_status(runner.STATUS_DISABLED)) == len(test_names):
+ if len(successful) + disabled == len(test_names):
self._stream.writeln('All tests successfully passed!')
return Manager.SUCCESS
_______________________________________________
webkit-changes mailing list
[email protected]
https://lists.webkit.org/mailman/listinfo/webkit-changes
