Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py (137355 => 137356)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py 2012-12-11 21:42:05 UTC (rev 137355)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py 2012-12-11 21:45:13 UTC (rev 137356)
@@ -260,8 +260,7 @@
self._batch_count = None
self._filesystem = None
self._driver = None
- self._tests_run_file = None
- self._tests_run_filename = None
+ self._num_tests = 0
def __del__(self):
self.stop()
@@ -276,8 +275,6 @@
self._batch_count = 0
self._batch_size = self._options.batch_size or 0
- tests_run_filename = self._filesystem.join(self._results_directory, "tests_run%d.txt" % self._worker_number)
- self._tests_run_file = self._filesystem.open_text_file_for_writing(tests_run_filename)
def handle(self, name, source, test_list_name, test_inputs):
assert name == 'test_list'
@@ -311,6 +308,8 @@
result.shard_name = shard_name
result.worker_name = self._name
result.total_run_time = time.time() - start
+ result.test_number = self._num_tests
+ self._num_tests += 1
self._caller.post('finished_test', result)
@@ -319,9 +318,6 @@
def stop(self):
_log.debug("%s cleaning up" % self._name)
self._kill_driver()
- if self._tests_run_file:
- self._tests_run_file.close()
- self._tests_run_file = None
def _timeout(self, test_input):
"""Compute the appropriate timeout value for a test."""
@@ -355,7 +351,6 @@
def _clean_up_after_test(self, test_input, result):
test_name = test_input.test_name
- self._tests_run_file.write(test_name + "\n")
if result.failures:
# Check and kill DumpRenderTree if we need to.
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (137355 => 137356)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2012-12-11 21:42:05 UTC (rev 137355)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2012-12-11 21:45:13 UTC (rev 137356)
@@ -34,6 +34,7 @@
create a final report.
"""
+import json
import logging
import random
import sys
@@ -476,10 +477,15 @@
"""
_log.debug("Writing JSON files in %s." % self._results_directory)
+ # FIXME: Upload stats.json to the server and delete times_ms.
times_trie = json_results_generator.test_timings_trie(self._port, result_summary.results.values())
times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
json_results_generator.write_json(self._filesystem, times_trie, times_json_path)
+ stats_trie = self._stats_trie(result_summary)
+ stats_path = self._filesystem.join(self._results_directory, "stats.json")
+ self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
+
full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
# We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
json_results_generator.write_json(self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS")
@@ -495,6 +501,7 @@
_log.debug("Finished writing JSON files.")
+
json_files = ["incremental_results.json", "full_results.json", "times_ms.json"]
generator.upload_json_files(json_files)
@@ -513,3 +520,16 @@
# so make sure it exists before we try to copy it.
if self._filesystem.exists(results_file):
self._filesystem.copyfile(results_file, destination_path)
+
+ def _stats_trie(self, result_summary):
+ def _worker_number(worker_name):
+ return int(worker_name.split('/')[1]) if worker_name else -1
+
+ stats = {}
+ for result in result_summary.results.values():
+ if result.type != test_expectations.SKIP:
+ stats[result.test_name] = {'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(result.test_run_time * 1000), int(result.total_run_time * 1000))}
+ stats_trie = {}
+ for name, value in stats.iteritems():
+ json_results_generator.add_path_to_trie(name, value, stats_trie)
+ return stats_trie
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (137355 => 137356)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-12-11 21:42:05 UTC (rev 137355)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-12-11 21:45:13 UTC (rev 137356)
@@ -578,7 +578,6 @@
'failures/unexpected/text-image-checksum.html'],
tests_included=True, host=host)
file_list = host.filesystem.written_files.keys()
- file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEqual(res, 1)
expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"IMAGE+TEXT","image_diff_percent":1},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
@@ -727,7 +726,6 @@
self.assertEqual(res, 0)
self.assertTrue('Retrying' in err.getvalue())
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
- self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/tests_run0.txt'))
self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/retries/failures/flaky/text-actual.txt'))
# Now we test that --clobber-old-results does remove the old entries and the old retries,
@@ -914,13 +912,18 @@
compressed_results = json.loads(json_to_eval)
return compressed_results
+ # Check that we recorded the test run times and ordering. Note that
+ # pretty much none of the actual values can be guaranteed, but at least
+ # we can test that they're there.
+ stats = json.loads(host.filesystem.read_text_file('/tmp/layout-test-results/stats.json'))
+ self.assertEqual(len(stats['http']['tests']['passes']['image.html']['results']), 5)
+
def test_reftest_with_two_notrefs(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
host = MockHost()
res, _, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host)
file_list = host.filesystem.written_files.keys()
- file_list.remove('/tmp/layout-test-results/tests_run0.txt')
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
json = self.parse_full_results(json_string)
self.assertTrue("multiple-match-success.html" not in json["tests"]["reftests"]["foo"])
@@ -954,9 +957,8 @@
['--pixel-tests', '--reset-results', 'passes/image.html', 'failures/expected/missing_image.html'],
tests_included=True, host=host, new_results=True)
file_list = host.filesystem.written_files.keys()
- file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEqual(res, 0)
- self.assertEqual(len(file_list), 7)
+ self.assertEqual(len(file_list), 8)
self.assertBaselines(file_list, "passes/image", [".txt", ".png"], err)
self.assertBaselines(file_list, "failures/expected/missing_image", [".txt", ".png"], err)
@@ -971,9 +973,8 @@
'failures/unexpected/missing_render_tree_dump.html'],
tests_included=True, host=host, new_results=True)
file_list = host.filesystem.written_files.keys()
- file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEqual(res, 0)
- self.assertEqual(len(file_list), 9)
+ self.assertEqual(len(file_list), 10)
self.assertBaselines(file_list, "failures/unexpected/missing_text", [".txt"], err)
self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_image", [".png"], err)
self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_render_tree_dump", [".txt"], err)
@@ -986,9 +987,8 @@
['--pixel-tests', '--new-baseline', 'passes/image.html', 'failures/expected/missing_image.html'],
tests_included=True, host=host, new_results=True)
file_list = host.filesystem.written_files.keys()
- file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEqual(res, 0)
- self.assertEqual(len(file_list), 7)
+ self.assertEqual(len(file_list), 8)
self.assertBaselines(file_list,
"platform/test-mac-leopard/passes/image", [".txt", ".png"], err)
self.assertBaselines(file_list,
