Diff
Modified: trunk/Tools/ChangeLog (89853 => 89854)
--- trunk/Tools/ChangeLog 2011-06-27 21:19:10 UTC (rev 89853)
+++ trunk/Tools/ChangeLog 2011-06-27 21:20:01 UTC (rev 89854)
@@ -2,6 +2,23 @@
Reviewed by Dirk Pranke.
+ new-run-webkit-tests does not report stderr output
+ https://bugs.webkit.org/show_bug.cgi?id=37739
+
+ Almost all the infrastructure appears to be in place to report stderr
+ in results.html. We just need to wire up a bool to tell results.html
+ to create the hyperlink.
+
+ * Scripts/webkitpy/layout_tests/layout_package/manager.py:
+ * Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py:
+ * Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py:
+ * Scripts/webkitpy/layout_tests/layout_package/test_results.py:
+ * Scripts/webkitpy/layout_tests/port/base.py:
+
+2011-06-27 Adam Barth <[email protected]>
+
+ Reviewed by Dirk Pranke.
+
webkitpy unit tests should have more descriptive names than just "Test"
https://bugs.webkit.org/show_bug.cgi?id=63479
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/manager.py (89853 => 89854)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/manager.py 2011-06-27 21:19:10 UTC (rev 89853)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/manager.py 2011-06-27 21:20:01 UTC (rev 89854)
@@ -132,7 +132,7 @@
continue
test_dict = {}
- if False:
+ if result.has_stderr:
test_dict['has_stderr'] = True
if result_type == test_expectations.PASS:
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py (89853 => 89854)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py 2011-06-27 21:19:10 UTC (rev 89853)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py 2011-06-27 21:20:01 UTC (rev 89854)
@@ -140,7 +140,7 @@
# FIXME: It the test crashed or timed out, it might be bettter to avoid
# to write new baselines.
self._save_baselines(driver_output)
- return TestResult(self._filename, failures, driver_output.test_time)
+ return TestResult(self._filename, failures, driver_output.test_time, driver_output.has_stderr())
def _save_baselines(self, driver_output):
# Although all test_shell/DumpRenderTree output should be utf-8,
@@ -220,13 +220,13 @@
if driver_output.crash:
# Don't continue any more if we already have a crash.
# In case of timeouts, we continue since we still want to see the text and image output.
- return TestResult(self._filename, failures, driver_output.test_time)
+ return TestResult(self._filename, failures, driver_output.test_time, driver_output.has_stderr())
failures.extend(self._compare_text(driver_output.text, expected_driver_output.text))
failures.extend(self._compare_audio(driver_output.audio, expected_driver_output.audio))
if self._options.pixel_tests:
failures.extend(self._compare_image(driver_output, expected_driver_output))
- return TestResult(self._filename, failures, driver_output.test_time)
+ return TestResult(self._filename, failures, driver_output.test_time, driver_output.has_stderr())
def _compare_text(self, actual_text, expected_text):
failures = []
@@ -281,18 +281,19 @@
def _compare_output_with_reference(self, driver_output1, driver_output2):
total_test_time = driver_output1.test_time + driver_output2.test_time
+ has_stderr = driver_output1.has_stderr() or driver_output2.has_stderr()
failures = []
failures.extend(self._handle_error(driver_output1))
if failures:
# Don't continue any more if we already have crash or timeout.
- return TestResult(self._filename, failures, total_test_time)
+ return TestResult(self._filename, failures, total_test_time, has_stderr)
failures.extend(self._handle_error(driver_output2, reference_filename=self._reference_filename))
if failures:
- return TestResult(self._filename, failures, total_test_time)
+ return TestResult(self._filename, failures, total_test_time, has_stderr)
if self._is_mismatch_reftest:
if driver_output1.image_hash == driver_output2.image_hash:
failures.append(test_failures.FailureReftestMismatchDidNotOccur())
elif driver_output1.image_hash != driver_output2.image_hash:
failures.append(test_failures.FailureReftestMismatch())
- return TestResult(self._filename, failures, total_test_time)
+ return TestResult(self._filename, failures, total_test_time, has_stderr)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py (89853 => 89854)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py 2011-06-27 21:19:10 UTC (rev 89853)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py 2011-06-27 21:20:01 UTC (rev 89854)
@@ -99,6 +99,7 @@
FILENAME_SUFFIX_ACTUAL = "-actual"
FILENAME_SUFFIX_EXPECTED = "-expected"
FILENAME_SUFFIX_DIFF = "-diff"
+ FILENAME_SUFFIX_STDERR = "-stderr"
FILENAME_SUFFIX_WDIFF = "-wdiff.html"
FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html"
FILENAME_SUFFIX_IMAGE_DIFF = "-diff.png"
@@ -161,7 +162,7 @@
def write_stderr(self, error):
fs = self._port._filesystem
- filename = self.output_filename("-stderr.txt")
+ filename = self.output_filename(self.FILENAME_SUFFIX_STDERR + ".txt")
fs.maybe_make_directory(fs.dirname(filename))
fs.write_binary_file(filename, error)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py (89853 => 89854)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py 2011-06-27 21:19:10 UTC (rev 89853)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py 2011-06-27 21:20:01 UTC (rev 89854)
@@ -38,11 +38,12 @@
def loads(str):
return cPickle.loads(str)
- def __init__(self, filename, failures=None, test_run_time=None):
+ def __init__(self, filename, failures=None, test_run_time=None, has_stderr=False):
# FIXME: s/filename/name to be consistent with the rest of layout_package.
self.filename = filename
self.failures = failures or []
self.test_run_time = test_run_time or 0
+ self.has_stderr = has_stderr
# FIXME: Setting this in the constructor makes this class hard to mutate.
self.type = test_failures.determine_result_type(failures)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py (89853 => 89854)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py 2011-06-27 21:19:10 UTC (rev 89853)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py 2011-06-27 21:20:01 UTC (rev 89854)
@@ -961,7 +961,10 @@
self.timeout = timeout
self.error = error
+ def has_stderr(self):
+ return bool(self.error)
+
class Driver:
"""Abstract interface for the DumpRenderTree interface."""
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py (89853 => 89854)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2011-06-27 21:19:10 UTC (rev 89853)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2011-06-27 21:20:01 UTC (rev 89854)
@@ -137,6 +137,8 @@
expected_text="foo\r\r\r\n", actual_text="foo\n")
tests.add('failures/expected/text.html', actual_text='text_fail-png')
tests.add('failures/unexpected/crash.html', crash=True)
+ tests.add('failures/unexpected/crash-with-stderr.html', crash=True,
+ error="mock-std-error-output")
tests.add('failures/unexpected/text-image-checksum.html',
actual_text='text-image-checksum_fail-txt',
actual_checksum='text-image-checksum_fail-checksum')
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (89853 => 89854)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2011-06-27 21:19:10 UTC (rev 89853)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2011-06-27 21:20:01 UTC (rev 89854)
@@ -348,7 +348,7 @@
def test_run_singly_actually_runs_tests(self):
res, _, _, _ = logging_run(['--run-singly', 'failures/unexpected'])
- self.assertEquals(res, 5)
+ self.assertEquals(res, 6)
def test_single_file(self):
# FIXME: We should consider replacing more of the get_tests_run()-style tests
@@ -398,13 +398,23 @@
# Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
# FIXME: It's nice to have a routine in port/test.py that returns this number.
- unexpected_tests_count = 5
+ unexpected_tests_count = 6
self.assertEqual(res, unexpected_tests_count)
self.assertFalse(out.empty())
self.assertFalse(err.empty())
self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html'])
+ def test_crash_with_stderr(self):
+ fs = port.unit_test_filesystem()
+ res, buildbot_output, regular_output, user = logging_run([
+ 'failures/unexpected/crash-with-stderr.html',
+ ],
+ tests_included=True,
+ record_results=True,
+ filesystem=fs)
+ self.assertTrue(fs.read_text_file('/tmp/layout-test-results/unexpected_results.json').find('{"crash-with-stderr.html":{"expected":"PASS","actual":"CRASH","has_stderr":true}}') != -1)
+
def test_exit_after_n_failures_upload(self):
fs = port.unit_test_filesystem()
res, buildbot_output, regular_output, user = logging_run([
