- Revision
- 225262
- Author
- [email protected]
- Date
- 2017-11-28 23:25:53 -0800 (Tue, 28 Nov 2017)
Log Message
WebDriver: add an option to dump test results to a json file
https://bugs.webkit.org/show_bug.cgi?id=180082
Reviewed by Brian Burg.
Add --json-output command line option to run-webdriver-tests to dump test results to a json file in a format
compatible with the W3C report. WebDriverTestResult now represents a test file and contains a list of
subtests, instead of having one WebDriverTestResult per subtest. This way we can store also the harness result
and dump the results to different formats.
* Scripts/run-webdriver-tests:
* Scripts/webkitpy/webdriver_tests/webdriver_test_result.py:
(WebDriverTestResult.__init__):
(WebDriverTestResult):
(WebDriverTestResult.add_subtest_results):
* Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py:
(WebDriverTestRunner.print_results):
(WebDriverTestRunner):
(WebDriverTestRunner.dump_results_to_json_file):
* Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
(WebDriverTestRunnerW3C.run):
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (225261 => 225262)
--- trunk/Tools/ChangeLog 2017-11-29 07:09:15 UTC (rev 225261)
+++ trunk/Tools/ChangeLog 2017-11-29 07:25:53 UTC (rev 225262)
@@ -1,3 +1,27 @@
+2017-11-28 Carlos Garcia Campos <[email protected]>
+
+ WebDriver: add an option to dump test results to a json file
+ https://bugs.webkit.org/show_bug.cgi?id=180082
+
+ Reviewed by Brian Burg.
+
+ Add --json-output command line option to run-webdriver-tests to dump test results to a json file in a format
+ compatible with the W3C report. WebDriverTestResult now represents a test file and contains a list of
+ subtests, instead of having one WebDriverTestResult per subtest. This way we can store also the harness result
+ and dump the results to different formats.
+
+ * Scripts/run-webdriver-tests:
+ * Scripts/webkitpy/webdriver_tests/webdriver_test_result.py:
+ (WebDriverTestResult.__init__):
+ (WebDriverTestResult):
+ (WebDriverTestResult.add_subtest_results):
+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py:
+ (WebDriverTestRunner.print_results):
+ (WebDriverTestRunner):
+ (WebDriverTestRunner.dump_results_to_json_file):
+ * Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py:
+ (WebDriverTestRunnerW3C.run):
+
2017-11-28 Alexey Proskuryakov <[email protected]>
Stop silencing leaks in TextCodecICU::registerCodecs, as the problem was fixed a while ago.
Modified: trunk/Tools/Scripts/run-webdriver-tests (225261 => 225262)
--- trunk/Tools/Scripts/run-webdriver-tests 2017-11-29 07:09:15 UTC (rev 225261)
+++ trunk/Tools/Scripts/run-webdriver-tests 2017-11-29 07:25:53 UTC (rev 225262)
@@ -47,6 +47,8 @@
help='Set the configuration to Debug')
option_parser.add_option('--timeout', action='', type='int', dest='timeout', default=10,
help='Time in seconds until a test times out (use 0 to disable)')
+option_parser.add_option('--json-output', action='', metavar="FILE",
+ help='Write results to JSON file at the given path')
option_parser.add_option('--display-server', choices=['xvfb', 'xorg', 'weston', 'wayland'], default='xvfb',
help='"xvfb": Use a virtualized X11 server. "xorg": Use the current X11 session. '
'"weston": Use a virtualized Weston server. "wayland": Use the current wayland session.')
@@ -66,5 +68,8 @@
retval = runner.run(args)
runner.print_results()
+if options.json_output is not None:
+ runner.dump_results_to_json_file(options.json_output)
+
sys.exit(retval)
Modified: trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_result.py (225261 => 225262)
--- trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_result.py 2017-11-29 07:09:15 UTC (rev 225261)
+++ trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_result.py 2017-11-29 07:25:53 UTC (rev 225262)
@@ -25,11 +25,14 @@
class WebDriverTestResult(object):
- def __init__(self, test_prefix, test, status, message, backtrace=None):
- self.test = os.path.join(test_prefix, test)
+ def __init__(self, test, status, message=None):
+ self.test = test
self.status = status
self.message = message
- self.backtrace = backtrace
+ self.subtest_results = []
+ def add_subtest_results(self, subtest, status, message, backtrace):
+ self.subtest_results.append((subtest, status, message, backtrace))
+
def __repr__(self):
return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.test, self.status)
Modified: trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py (225261 => 225262)
--- trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py 2017-11-29 07:09:15 UTC (rev 225261)
+++ trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner.py 2017-11-29 07:25:53 UTC (rev 225262)
@@ -20,7 +20,9 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import json
import logging
+import os
from webkitpy.webdriver_tests.webdriver_test_runner_w3c import WebDriverTestRunnerW3C
@@ -54,11 +56,16 @@
passed_count = 0
failures_count = 0
for result in self._runner.results():
- if result.status == 'PASS':
- passed_count += 1
- elif result.status == 'FAIL':
- results.setdefault(result.status, []).append(result.test)
- failures_count += 1
+ if result.status == 'OK':
+ for subtest, status, _, _ in result.subtest_results:
+ if status == 'PASS':
+ passed_count += 1
+ elif status == 'FAIL':
+ results.setdefault(status, []).append(os.path.join(os.path.dirname(result.test), subtest))
+ failures_count += 1
+ else:
+ # FIXME: handle other results.
+ pass
_log.info('')
@@ -73,3 +80,27 @@
_log.info('Unexpected failures (%d)' % len(failed))
for test in failed:
_log.info(' %s' % test)
+
+ def dump_results_to_json_file(self, output_path):
+ json_results = {}
+ json_results['results'] = []
+ for result in self._runner.results():
+ results = {}
+ results['test'] = result.test
+ results['status'] = result.status
+ results['message'] = result.message
+ results['subtests'] = []
+ for name, status, message, _ in result.subtest_results:
+ subtest = {}
+ subtest['name'] = name
+ subtest['status'] = status
+ subtest['message'] = message
+ results['subtests'].append(subtest)
+ json_results['results'].append(results)
+
+ directory = os.path.dirname(output_path)
+ if not os.path.exists(directory):
+ os.makedirs(directory)
+
+ with open(output_path, 'wb') as fp:
+ json.dump(json_results, fp)
Modified: trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py (225261 => 225262)
--- trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py 2017-11-29 07:09:15 UTC (rev 225261)
+++ trunk/Tools/Scripts/webkitpy/webdriver_tests/webdriver_test_runner_w3c.py 2017-11-29 07:25:53 UTC (rev 225262)
@@ -96,10 +96,14 @@
for test in tests:
test_name = os.path.relpath(test, self._tests_dir())
harness_result, test_results = executor.run(test)
- if harness_result[0] != 'OK':
- _log.error("Failed to run test %s: %s" % (test_name, harness_result[1]))
+ result = WebDriverTestResult(test_name, *harness_result)
+ if harness_result[0] == 'OK':
+ for test_result in test_results:
+ result.add_subtest_results(*test_result)
else:
- self._add_results(os.path.dirname(test_name), test_results)
+ # FIXME: handle other results.
+ pass
+ self._results.append(result)
finally:
executor.teardown()
self._server.stop()
