Diff
Modified: trunk/Tools/ChangeLog (90054 => 90055)
--- trunk/Tools/ChangeLog 2011-06-29 22:24:48 UTC (rev 90054)
+++ trunk/Tools/ChangeLog 2011-06-29 22:25:58 UTC (rev 90055)
@@ -1,3 +1,30 @@
+2011-06-29 Adam Barth <[email protected]>
+
+ Reviewed by Dirk Pranke.
+
+ new-run-webkit-tests complains about missing pixel results instead of plopping down new expectations
+ https://bugs.webkit.org/show_bug.cgi?id=38063
+
+ This patch changes new-run-webkit-tests to match old-run-webkit-tests
+ in generating new expected results when the expected results are
+ missing.
+
+ There are still a couple details that are different:
+
+ 1) Image baselines aren't generated unless you run with -p.
+ 2) Render tree dumps are places in the cross-platform directory instead
+ of the platform-specific directory.
+
+ I'm inclined to deal with both of these issues in follow-up patches.
+
+ * Scripts/webkitpy/common/net/layouttestresults.py:
+ * Scripts/webkitpy/layout_tests/layout_package/manager.py:
+ * Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py:
+ * Scripts/webkitpy/layout_tests/layout_package/test_results.py:
+ * Scripts/webkitpy/layout_tests/port/test.py:
+ * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+
2011-06-29 Adam Roben <[email protected]>
Teach TestFailures to detect possibly flaky tests and list them separately
Modified: trunk/Tools/Scripts/webkitpy/common/net/layouttestresults.py (90054 => 90055)
--- trunk/Tools/Scripts/webkitpy/common/net/layouttestresults.py 2011-06-29 22:24:48 UTC (rev 90054)
+++ trunk/Tools/Scripts/webkitpy/common/net/layouttestresults.py 2011-06-29 22:25:58 UTC (rev 90055)
@@ -161,7 +161,7 @@
return self._test_results
def results_matching_failure_types(self, failure_types):
- return [result for result in self._test_results if result.has_failure_matching_types(failure_types)]
+ return [result for result in self._test_results if result.has_failure_matching_types(*failure_types)]
def tests_matching_failure_types(self, failure_types):
return [result.filename for result in self.results_matching_failure_types(failure_types)]
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/manager.py (90054 => 90055)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/manager.py 2011-06-29 22:24:48 UTC (rev 90054)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/manager.py 2011-06-29 22:25:58 UTC (rev 90055)
@@ -795,7 +795,7 @@
# We exclude the crashes from the list of results to retry, because
# we want to treat even a potentially flaky crash as an error.
- failures = self._get_failures(result_summary, include_crashes=False)
+ failures = self._get_failures(result_summary, include_crashes=False, include_missing=False)
retry_summary = result_summary
while (len(failures) and self._options.retry_failures and
not self._retrying and not interrupted):
@@ -806,7 +806,7 @@
retry_summary = ResultSummary(self._expectations, failures.keys())
# Note that we intentionally ignore the return value here.
self._run_tests(failures.keys(), retry_summary)
- failures = self._get_failures(retry_summary, include_crashes=True)
+ failures = self._get_failures(retry_summary, include_crashes=True, include_missing=True)
end_time = time.time()
@@ -933,7 +933,7 @@
if self._fs.isdir(self._fs.join(layout_tests_dir, dirname)):
self._fs.rmtree(self._fs.join(self._results_directory, dirname))
- def _get_failures(self, result_summary, include_crashes):
+ def _get_failures(self, result_summary, include_crashes, include_missing):
"""Filters a dict of results and returns only the failures.
Args:
@@ -948,7 +948,8 @@
failed_results = {}
for test, result in result_summary.unexpected_results.iteritems():
if (result.type == test_expectations.PASS or
- result.type == test_expectations.CRASH and not include_crashes):
+ (result.type == test_expectations.CRASH and not include_crashes) or
+ (result.type == test_expectations.MISSING and not include_missing)):
continue
failed_results[test] = result.type
@@ -1297,7 +1298,7 @@
if self._options.full_results_html:
test_files = result_summary.failures.keys()
else:
- unexpected_failures = self._get_failures(result_summary, include_crashes=True)
+ unexpected_failures = self._get_failures(result_summary, include_crashes=True, include_missing=True)
test_files = unexpected_failures.keys()
if not len(test_files):
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py (90054 => 90055)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py 2011-06-29 22:24:48 UTC (rev 90054)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py 2011-06-29 22:25:58 UTC (rev 90055)
@@ -128,31 +128,40 @@
driver_output = self._driver.run_test(self._driver_input())
expected_driver_output = self._expected_driver_output()
test_result = self._compare_output(driver_output, expected_driver_output)
- test_result_writer.write_test_result(self._port, self._filename,
- driver_output, expected_driver_output, test_result.failures)
+ if self._options.new_test_results:
+ self._add_missing_baselines(test_result, driver_output)
+ test_result_writer.write_test_result(self._port, self._filename, driver_output, expected_driver_output, test_result.failures)
return test_result
def _run_rebaseline(self):
driver_output = self._driver.run_test(self._driver_input())
failures = self._handle_error(driver_output)
- test_result_writer.write_test_result(self._port, self._filename,
- driver_output, None, failures)
+ test_result_writer.write_test_result(self._port, self._filename, driver_output, None, failures)
# FIXME: It the test crashed or timed out, it might be bettter to avoid
# to write new baselines.
- self._save_baselines(driver_output)
+ self._overwrite_baselines(driver_output)
return TestResult(self._filename, failures, driver_output.test_time, driver_output.has_stderr())
- def _save_baselines(self, driver_output):
+ def _add_missing_baselines(self, test_result, driver_output):
+ if test_result.has_failure_matching_types(test_failures.FailureMissingResult):
+ # FIXME: We seem to be putting new text results in non-platform
+ # specific directories even when they're rendertree dumps. Maybe
+ # we should have a different kind of failure for render tree dumps
+ # than for text tests?
+ self._save_baseline_data(driver_output.text, ".txt", generate_new_baseline=False)
+ if test_result.has_failure_matching_types(test_failures.FailureMissingAudio):
+ self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=False)
+ if test_result.has_failure_matching_types(test_failures.FailureMissingImage, test_failures.FailureMissingImageHash):
+ self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=False)
+
+ def _overwrite_baselines(self, driver_output):
# Although all DumpRenderTree output should be utf-8,
# we do not ever decode it inside run-webkit-tests. For some tests
# DumpRenderTree may not output utf-8 text (e.g. webarchives).
- self._save_baseline_data(driver_output.text, ".txt",
- generate_new_baseline=self._options.new_baseline)
- self._save_baseline_data(driver_output.audio, '.wav',
- generate_new_baseline=self._options.new_baseline)
+ self._save_baseline_data(driver_output.text, ".txt", generate_new_baseline=self._options.new_baseline)
+ self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=self._options.new_baseline)
if self._options.pixel_tests:
- self._save_baseline_data(driver_output.image, ".png",
- generate_new_baseline=self._options.new_baseline)
+ self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=self._options.new_baseline)
def _save_baseline_data(self, data, modifier, generate_new_baseline=True):
"""Saves a new baseline file into the port's baseline directory.
@@ -178,10 +187,10 @@
"-expected" + modifier)
fs.maybe_make_directory(output_dir)
output_path = fs.join(output_dir, output_file)
- _log.debug('writing new baseline result "%s"' % (output_path))
+ _log.debug('Writing new baseline result "%s"' % output_path)
else:
output_path = port.expected_filename(self._filename, modifier)
- _log.debug('resetting baseline result "%s"' % output_path)
+ _log.debug('Resetting baseline result "%s"' % output_path)
port.update_baseline(output_path, data)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py (90054 => 90055)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py 2011-06-29 22:24:48 UTC (rev 90054)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py 2011-06-29 22:25:58 UTC (rev 90055)
@@ -55,9 +55,9 @@
def __ne__(self, other):
return not (self == other)
- def has_failure_matching_types(self, types):
+ def has_failure_matching_types(self, *args, **kargs):
for failure in self.failures:
- if type(failure) in types:
+ if type(failure) in args:
return True
return False
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py (90054 => 90055)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2011-06-29 22:24:48 UTC (rev 90054)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2011-06-29 22:25:58 UTC (rev 90055)
@@ -136,6 +136,8 @@
tests.add('failures/expected/newlines_with_excess_CR.html',
expected_text="foo\r\r\r\n", actual_text="foo\n")
tests.add('failures/expected/text.html', actual_text='text_fail-png')
+ tests.add('failures/unexpected/missing_text.html', expected_text=None)
+ tests.add('failures/unexpected/missing_image.html', expected_image=None)
tests.add('failures/unexpected/crash.html', crash=True)
tests.add('failures/unexpected/crash-with-stderr.html', crash=True,
error="mock-std-error-output")
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py (90054 => 90055)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py 2011-06-29 22:24:48 UTC (rev 90054)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py 2011-06-29 22:25:58 UTC (rev 90055)
@@ -279,8 +279,6 @@
# FIXME: Need: --sample-on-timeout Run sample on timeout
old_run_webkit_tests_compat = [
- # NRWT doesn't generate results by default anyway.
- _compat_shim_option("--no-new-test-results"),
# NRWT doesn't sample on timeout yet anyway.
_compat_shim_option("--no-sample-on-timeout"),
# FIXME: NRWT needs to support remote links eventually.
@@ -307,6 +305,9 @@
optparse.make_option("--reset-results", action=""
default=False, help="Reset any existing baselines to the "
"generated results"),
+ optparse.make_option("--no-new-test-results", action=""
+ dest="new_test_results", default=True,
+ help="Don't create new baselines when no expected results exist"),
optparse.make_option("--skip-failing-tests", action=""
default=False, help="Skip tests that are expected to fail. "
"Note: When using this option, you might miss new crashes "
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (90054 => 90055)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2011-06-29 22:24:48 UTC (rev 90054)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2011-06-29 22:25:58 UTC (rev 90055)
@@ -641,9 +641,9 @@
class RebaselineTest(unittest.TestCase):
- def assertBaselines(self, file_list, file):
+ def assertBaselines(self, file_list, file, extensions):
"assert that the file_list contains the baselines."""
- for ext in (".txt", ".png"):
+ for ext in extensions:
baseline = file + "-expected" + ext
self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
@@ -662,9 +662,24 @@
file_list = fs.written_files.keys()
file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEqual(len(file_list), 4)
- self.assertBaselines(file_list, "/passes/image")
- self.assertBaselines(file_list, "/failures/expected/missing_image")
+ self.assertBaselines(file_list, "/passes/image", [".txt", ".png"])
+ self.assertBaselines(file_list, "/failures/expected/missing_image", [".txt", ".png"])
+ def test_missing_results(self):
+ # Test that we update expectations in place. If the expectation
+ # is missing, update the expected generic location.
+ fs = port.unit_test_filesystem()
+ passing_run(['--no-show-results',
+ 'failures/unexpected/missing_text.html',
+ 'failures/unexpected/missing_image.html',
+ 'failures/unexpected/missing_audio.html'],
+ tests_included=True, filesystem=fs)
+ file_list = fs.written_files.keys()
+ file_list.remove('/tmp/layout-test-results/tests_run0.txt')
+ self.assertEqual(len(file_list), 4)
+ self.assertBaselines(file_list, "/failures/unexpected/missing_text", [".txt"])
+ self.assertBaselines(file_list, "/failures/unexpected/missing_image", [".png"])
+
def test_new_baseline(self):
# Test that we update the platform expectations. If the expectation
# is mssing, then create a new expectation in the platform dir.
@@ -678,9 +693,9 @@
file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEqual(len(file_list), 4)
self.assertBaselines(file_list,
- "/platform/test-mac-leopard/passes/image")
+ "/platform/test-mac-leopard/passes/image", [".txt", ".png"])
self.assertBaselines(file_list,
- "/platform/test-mac-leopard/failures/expected/missing_image")
+ "/platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"])
class DryrunTest(unittest.TestCase):
