- Revision
- 235252
- Author
- [email protected]
- Date
- 2018-08-23 15:50:52 -0700 (Thu, 23 Aug 2018)
Log Message
API tests should output json results
https://bugs.webkit.org/show_bug.cgi?id=188869
<rdar://problem/43615652>
Reviewed by Aakash Jain.
JSON output for API tests is of the form:
{
"Failed": [{"name": <test name>, "output": <test log>}],
"Timedout": [...],
"Skipped": [...],
"Crashed": [...]
}
Tests which are successful are not displayed in the json output.
* Scripts/webkitpy/api_tests/manager.py:
(Manager.run): Print test results to provided file as a json dictionary.
* Scripts/webkitpy/api_tests/run_api_tests.py:
(run): Pass json option.
(parse_args): Add --json-output flag.
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (235251 => 235252)
--- trunk/Tools/ChangeLog 2018-08-23 22:23:18 UTC (rev 235251)
+++ trunk/Tools/ChangeLog 2018-08-23 22:50:52 UTC (rev 235252)
@@ -1,3 +1,26 @@
+2018-08-23 Jonathan Bedard <[email protected]>
+
+ API tests should output json results
+ https://bugs.webkit.org/show_bug.cgi?id=188869
+ <rdar://problem/43615652>
+
+ Reviewed by Aakash Jain.
+
+ JSON output for API tests is of the form:
+ {
+ "Failed": [{"name": <test name>, "output": <test log>}],
+ "Timedout": [...],
+ "Skipped": [...],
+ "Crashed": [...]
+ }
+ Tests which are successful are not displayed in the json output.
+
+ * Scripts/webkitpy/api_tests/manager.py:
+ (Manager.run): Print test results to provided file as a json dictionary.
+ * Scripts/webkitpy/api_tests/run_api_tests.py:
+ (run): Pass json option.
+ (parse_args): Add --json-output flag.
+
2018-08-23 Andy Estes <[email protected]>
[Apple Pay] Introduce Apple Pay JS v4 on iOS 12 and macOS Mojave
Modified: trunk/Tools/Scripts/webkitpy/api_tests/manager.py (235251 => 235252)
--- trunk/Tools/Scripts/webkitpy/api_tests/manager.py 2018-08-23 22:23:18 UTC (rev 235251)
+++ trunk/Tools/Scripts/webkitpy/api_tests/manager.py 2018-08-23 22:50:52 UTC (rev 235252)
@@ -20,6 +20,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import json
import logging
import os
@@ -148,7 +149,12 @@
return self._port.path_to_api_test_binaries().keys()
return binaries or self._port.path_to_api_test_binaries().keys()
- def run(self, args):
+ def run(self, args, json_output=None):
+ if json_output:
+ json_output = self.host.filesystem.abspath(json_output)
+ if not self.host.filesystem.isdir(self.host.filesystem.dirname(json_output)) or self.host.filesystem.isdir(json_output):
+ raise RuntimeError('Cannot write to {}'.format(json_output))
+
self._stream.write_update('Checking build ...')
if not self._port.check_api_test_build(self._binaries_for_arguments(args)):
_log.error('Build check failed')
@@ -184,9 +190,18 @@
disabled = len(runner.result_map_by_status(runner.STATUS_DISABLED))
_log.info('Ran {} tests of {} with {} successful'.format(len(runner.results) - disabled, len(test_names), len(successful)))
- self._stream.writeln('------------------------------')
+ result_dictionary = {
+ 'Skipped': [],
+ 'Failed': [],
+ 'Crashed': [],
+ 'Timedout': [],
+ }
+
+ self._stream.writeln('-' * 30)
if len(successful) + disabled == len(test_names):
self._stream.writeln('All tests successfully passed!')
+ if json_output:
+ self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4))
return Manager.SUCCESS
self._stream.writeln('Test suite failed')
@@ -196,6 +211,7 @@
for test in test_names:
if test not in runner.results:
skipped.append(test)
+ result_dictionary['Skipped'].append({'name': test, 'output': None})
if skipped:
self._stream.writeln('Skipped {} tests'.format(len(skipped)))
self._stream.writeln('')
@@ -207,4 +223,17 @@
self._print_tests_result_with_status(runner.STATUS_CRASHED, runner)
self._print_tests_result_with_status(runner.STATUS_TIMEOUT, runner)
+ for test, result in runner.results.iteritems():
+ status_to_string = {
+ runner.STATUS_FAILED: 'Failed',
+ runner.STATUS_CRASHED: 'Crashed',
+ runner.STATUS_TIMEOUT: 'Timedout',
+ }.get(result[0])
+ if not status_to_string:
+ continue
+ result_dictionary[status_to_string].append({'name': test, 'output': result[1]})
+
+ if json_output:
+ self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4))
+
return Manager.FAILED_TESTS
Modified: trunk/Tools/Scripts/webkitpy/api_tests/run_api_tests.py (235251 => 235252)
--- trunk/Tools/Scripts/webkitpy/api_tests/run_api_tests.py 2018-08-23 22:23:18 UTC (rev 235251)
+++ trunk/Tools/Scripts/webkitpy/api_tests/run_api_tests.py 2018-08-23 22:50:52 UTC (rev 235252)
@@ -73,7 +73,7 @@
stream = MeteredStream(logging_stream, options.verbose, logger=logger, number_of_columns=port.host.platform.terminal_width(), print_timestamps=options.timestamps)
manager = Manager(port, options, stream)
- result = manager.run(args)
+ result = manager.run(args, json_output=options.json_output)
_log.debug("Testing completed, Exit status: %d" % result)
return result
finally:
@@ -92,6 +92,8 @@
help='Enable verbose printing'),
optparse.make_option('--timestamps', action='', default=False,
help='Print timestamps for each logged line'),
+ optparse.make_option('--json-output', action='', default=None,
+ help='Save test results as JSON to file'),
]))
option_group_definitions.append(('WebKit Options', [
