Diff
Modified: trunk/Tools/ChangeLog (120739 => 120740)
--- trunk/Tools/ChangeLog 2012-06-19 18:25:01 UTC (rev 120739)
+++ trunk/Tools/ChangeLog 2012-06-19 18:27:29 UTC (rev 120740)
@@ -1,5 +1,42 @@
2012-06-19 Dirk Pranke <[email protected]>
+ REGRESSION (NRWT): Results for new non-text-only tests are always put in the most-specific platform directory
+ https://bugs.webkit.org/show_bug.cgi?id=78127
+
+ Reviewed by Ryosuke Niwa.
+
+ Add the --add-platform-exceptions flag from ORWT, and ensure
+ that --new-baseline is equivalent to --reset-results
+ --add-platform-exceptions. Also fix the default logic for where to
+ put PNGs and render tree dumps if --new-test-results is true:
+ if --add-platform-exceptions is False, baselines should go in
+ the least-specific platform directory (e.g., platform/mac)
+ rather than the most-specific (platform/mac-snowleopard).
+
+ * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
+ (SingleTestRunner):
+ (SingleTestRunner.run):
+ (SingleTestRunner._run_rebaseline):
+ (SingleTestRunner._add_missing_baselines):
+ (SingleTestRunner._location_for_new_baseline):
+ (SingleTestRunner._overwrite_baselines):
+ (SingleTestRunner._save_baseline_data):
+ * Scripts/webkitpy/layout_tests/port/base.py:
+ (Port.baseline_path):
+ (Port):
+ (Port.baseline_platform_dir):
+ (Port.baseline_version_dir):
+ * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+ (_set_up_derived_options):
+ (parse_args):
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+ (RebaselineTest.assertBaselines):
+ (RebaselineTest.test_reset_results):
+ (RebaselineTest.test_missing_results):
+ (RebaselineTest.test_new_baseline):
+
+2012-06-19 Dirk Pranke <[email protected]>
+
NRWT should not take memory used as disk cache into account when deciding how many processes to launch
https://bugs.webkit.org/show_bug.cgi?id=81379
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py (120739 => 120740)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2012-06-19 18:25:01 UTC (rev 120739)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2012-06-19 18:27:29 UTC (rev 120740)
@@ -46,7 +46,8 @@
return runner.run()
-class SingleTestRunner:
+class SingleTestRunner(object):
+ (ALONGSIDE_TEST, PLATFORM_DIR, VERSION_DIR, UPDATE) = ('alongside', 'platform', 'version', 'update')
def __init__(self, options, port, driver, test_input, worker_name):
self._options = options
@@ -94,12 +95,12 @@
def run(self):
if self._reference_files:
- if self._port.get_option('no_ref_tests') or self._options.new_baseline or self._options.reset_results:
+ if self._port.get_option('no_ref_tests') or self._options.reset_results:
result = TestResult(self._test_name)
result.type = test_expectations.SKIP
return result
return self._run_reftest()
- if self._options.new_baseline or self._options.reset_results:
+ if self._options.reset_results:
return self._run_rebaseline()
return self._run_compare_test()
@@ -121,7 +122,7 @@
driver_output = self._driver.run_test(self._driver_input())
failures = self._handle_error(driver_output)
test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, driver_output, None, failures)
- # FIXME: It the test crashed or timed out, it might be bettter to avoid
+ # FIXME: It the test crashed or timed out, it might be better to avoid
# to write new baselines.
self._overwrite_baselines(driver_output)
return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr())
@@ -131,49 +132,50 @@
def _add_missing_baselines(self, test_result, driver_output):
missingImage = test_result.has_failure_matching_types(test_failures.FailureMissingImage, test_failures.FailureMissingImageHash)
if test_result.has_failure_matching_types(test_failures.FailureMissingResult):
- self._save_baseline_data(driver_output.text, ".txt", SingleTestRunner._render_tree_dump_pattern.match(driver_output.text))
+ self._save_baseline_data(driver_output.text, '.txt', self._location_for_new_baseline(driver_output.text, '.txt'))
if test_result.has_failure_matching_types(test_failures.FailureMissingAudio):
- self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=False)
+ self._save_baseline_data(driver_output.audio, '.wav', self._location_for_new_baseline(driver_output.audio, '.wav'))
if missingImage:
- self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=True)
+ self._save_baseline_data(driver_output.image, '.png', self._location_for_new_baseline(driver_output.image, '.png'))
+ def _location_for_new_baseline(self, data, extension):
+ if self._options.add_platform_exceptions:
+ return self.VERSION_DIR
+ if extension == '.png':
+ return self.PLATFORM_DIR
+ if extension == '.wav':
+ return self.ALONGSIDE_TEST
+ if extension == '.txt' and self._render_tree_dump_pattern.match(data):
+ return self.PLATFORM_DIR
+ return self.ALONGSIDE_TEST
+
def _overwrite_baselines(self, driver_output):
- # Although all DumpRenderTree output should be utf-8,
- # we do not ever decode it inside run-webkit-tests. For some tests
- # DumpRenderTree may not output utf-8 text (e.g. webarchives).
- self._save_baseline_data(driver_output.text, ".txt", generate_new_baseline=self._options.new_baseline)
- self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=self._options.new_baseline)
+ location = self.VERSION_DIR if self._options.add_platform_exceptions else self.UPDATE
+ self._save_baseline_data(driver_output.text, '.txt', location)
+ self._save_baseline_data(driver_output.audio, '.wav', location)
if self._options.pixel_tests:
- self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=self._options.new_baseline)
+ self._save_baseline_data(driver_output.image, '.png', location)
- def _save_baseline_data(self, data, modifier, generate_new_baseline=True):
- """Saves a new baseline file into the port's baseline directory.
-
- The file will be named simply "<test>-expected<modifier>", suitable for
- use as the expected results in a later run.
-
- Args:
- data: result to be saved as the new baseline
- modifier: type of the result file, e.g. ".txt" or ".png"
- generate_new_baseline: whether to enerate a new, platform-specific
- baseline, or update the existing one
- """
+ def _save_baseline_data(self, data, extension, location):
if data is None:
return
port = self._port
fs = self._filesystem
- if generate_new_baseline:
- relative_dir = fs.dirname(self._test_name)
- baseline_path = port.baseline_path()
- output_dir = fs.join(baseline_path, relative_dir)
- output_file = fs.basename(fs.splitext(self._test_name)[0] + "-expected" + modifier)
- fs.maybe_make_directory(output_dir)
- output_path = fs.join(output_dir, output_file)
+ if location == self.ALONGSIDE_TEST:
+ output_dir = fs.dirname(port.abspath_for_test(self._test_name))
+ elif location == self.VERSION_DIR:
+ output_dir = fs.join(port.baseline_version_dir(), fs.dirname(self._test_name))
+ elif location == self.PLATFORM_DIR:
+ output_dir = fs.join(port.baseline_platform_dir(), fs.dirname(self._test_name))
+ elif location == self.UPDATE:
+ output_dir = fs.dirname(port.expected_filename(self._test_name, extension))
else:
- output_path = port.expected_filename(self._test_name, modifier)
+ raise AssertionError('unrecognized baseline location: %s' % location)
- result_name = fs.relpath(output_path, port.layout_tests_dir())
- _log.info('Writing new expected result "%s"' % result_name)
+ fs.maybe_make_directory(output_dir)
+ output_basename = fs.basename(fs.splitext(self._test_name)[0] + "-expected" + extension)
+ output_path = fs.join(output_dir, output_basename)
+ _log.info('Writing new expected result "%s"' % port.relative_test_filename(output_path))
port.update_baseline(output_path, data)
def _handle_error(self, driver_output, reference_filename=None):
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py (120739 => 120740)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py 2012-06-19 18:25:01 UTC (rev 120739)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py 2012-06-19 18:27:29 UTC (rev 120740)
@@ -165,6 +165,15 @@
def baseline_path(self):
"""Return the absolute path to the directory to store new baselines in for this port."""
+ # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir()
+ return self.baseline_version_dir()
+
+ def baseline_platform_dir(self):
+ """Return the absolute path to the default (version-independent) platform-specific results."""
+ return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name)
+
+ def baseline_version_dir(self):
+ """Return the absolute path to the platform-and-version-specific results."""
baseline_search_paths = self.get_option('additional_platform_directory', []) + self.baseline_search_path()
return baseline_search_paths[0]
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py (120739 => 120740)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py 2012-06-19 18:25:01 UTC (rev 120739)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py 2012-06-19 18:27:29 UTC (rev 120740)
@@ -167,6 +167,10 @@
if options.ignore_metrics and (options.new_baseline or options.reset_results):
warnings.append("--ignore-metrics has no effect with --new-baselines or with --reset-results")
+ if options.new_baseline:
+ options.reset_results = True
+ options.add_platform_exceptions = True
+
return warnings
@@ -277,10 +281,12 @@
optparse.make_option("--results-directory", help="Location of test results"),
optparse.make_option("--build-directory",
help="Path to the directory under which build files are kept (should not include configuration)"),
+ optparse.make_option("--add-platform-exceptions", action="" default=False,
+ help="Save generated results into the *most-specific-platform* directory rather than the *generic-platform* directory"),
optparse.make_option("--new-baseline", action=""
default=False, help="Save generated results as new baselines "
- "into the *platform* directory, overwriting whatever's "
- "already there."),
+ "into the *most-specific-platform* directory, overwriting whatever's "
+ "already there. Equivalent to --reset-results --add-platform-exceptions"),
optparse.make_option("--reset-results", action=""
default=False, help="Reset expectations to the "
"generated results in their existing location."),
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (120739 => 120740)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-06-19 18:25:01 UTC (rev 120739)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-06-19 18:27:29 UTC (rev 120740)
@@ -930,7 +930,7 @@
"assert that the file_list contains the baselines."""
for ext in extensions:
baseline = file + "-expected" + ext
- baseline_msg = 'Writing new expected result "%s"\n' % baseline[1:]
+ baseline_msg = 'Writing new expected result "%s"\n' % baseline
self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
self.assertContainsLine(err, baseline_msg)
@@ -951,8 +951,8 @@
self.assertEquals(res, 0)
self.assertEmpty(out)
self.assertEqual(len(file_list), 4)
- self.assertBaselines(file_list, "/passes/image", [".txt", ".png"], err)
- self.assertBaselines(file_list, "/failures/expected/missing_image", [".txt", ".png"], err)
+ self.assertBaselines(file_list, "passes/image", [".txt", ".png"], err)
+ self.assertBaselines(file_list, "failures/expected/missing_image", [".txt", ".png"], err)
def test_missing_results(self):
# Test that we update expectations in place. If the expectation
@@ -969,13 +969,13 @@
self.assertEquals(res, 0)
self.assertNotEmpty(out)
self.assertEqual(len(file_list), 6)
- self.assertBaselines(file_list, "/failures/unexpected/missing_text", [".txt"], err)
- self.assertBaselines(file_list, "/platform/test-mac-leopard/failures/unexpected/missing_image", [".png"], err)
- self.assertBaselines(file_list, "/platform/test-mac-leopard/failures/unexpected/missing_render_tree_dump", [".txt"], err)
+ self.assertBaselines(file_list, "failures/unexpected/missing_text", [".txt"], err)
+ self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_image", [".png"], err)
+ self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_render_tree_dump", [".txt"], err)
def test_new_baseline(self):
- # Test that we update the platform expectations. If the expectation
- # is mssing, then create a new expectation in the platform dir.
+ # Test that we update the platform expectations in the version-specific directories
+ # for both existing and new baselines.
host = MockHost()
res, out, err, _ = logging_run(['--pixel-tests',
'--new-baseline',
@@ -988,9 +988,9 @@
self.assertEmpty(out)
self.assertEqual(len(file_list), 4)
self.assertBaselines(file_list,
- "/platform/test-mac-leopard/passes/image", [".txt", ".png"], err)
+ "platform/test-mac-leopard/passes/image", [".txt", ".png"], err)
self.assertBaselines(file_list,
- "/platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err)
+ "platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err)
if __name__ == '__main__':
