Diff
Modified: trunk/Tools/ChangeLog (270953 => 270954)
--- trunk/Tools/ChangeLog 2020-12-17 23:17:38 UTC (rev 270953)
+++ trunk/Tools/ChangeLog 2020-12-17 23:19:51 UTC (rev 270954)
@@ -1,3 +1,33 @@
+2020-12-17 Sam Sneddon <[email protected]>
+
+ Stop parsing reftest.list files
+ https://bugs.webkit.org/show_bug.cgi?id=203783
+
+ Reviewed by Jonathan Bedard.
+
+ This has only been used by the webkitpy integration tests in recent years. Needless to say,
+ this is Not Useful.
+
+ We do still have one reftest.list file,
+ media/track/opera/track/webvtt/rendering/reftest/reftest.list, but as this is in a reftest
+ directory we don't run any of the tests in it (and never have). See the above bug for more
+ detail about that history.
+
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+ (RebaselineTest.assertBaselines): docstring fix
+ (RunTest.test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist): Deleted.
+ (EndToEndTest): Deleted.
+ (EndToEndTest.test_reftest_with_two_notrefs): Deleted.
+ * Scripts/webkitpy/port/base.py:
+ (Port.__init__): remove Port._reftest_list
+ (Port.reference_files): don't use reftest.list as source of truth
+ (Port._get_reftest_list): Deleted.
+ (Port._parse_reftest_list): Deleted.
+ * Scripts/webkitpy/port/base_unittest.py:
+ (PortTest.test_parse_reftest_list): Deleted.
+ * Scripts/webkitpy/port/test.py:
+ (unit_test_list): remove tests used to test reftest.list
+
2020-12-17 Jonathan Bedard <[email protected]>
[run-webkit-tests] Replace FailureNotTested with FailureNoOutput
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (270953 => 270954)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2020-12-17 23:17:38 UTC (rev 270953)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2020-12-17 23:19:51 UTC (rev 270954)
@@ -807,70 +807,6 @@
tests_run = get_test_results(['passes/mismatch.html'])
self.assertEqual(tests_run[0].references, ['passes/mismatch-expected-mismatch.html'])
- def test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist(self):
- host = MockHost()
- _, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host)
- expected_dictionary = {
- 'version': 4,
- 'fixable': 5,
- 'skipped': 0,
- 'num_passes': 3,
- 'num_flaky': 0,
- 'num_missing': 1,
- 'num_regressions': 4,
- 'uses_expectations_file': True,
- 'interrupted': False,
- 'layout_tests_dir': '/test.checkout/LayoutTests',
- 'has_pretty_patch': False,
- 'pixel_tests_enabled': True,
- 'other_crashes': {},
- 'date': '10:27AM on December 13, 2019',
- 'tests': {
- 'reftests': {
- 'foo': {
- 'multiple-both-failure.html': {
- 'reftest_type': ['!=', '=='],
- 'report': 'REGRESSION',
- 'expected': 'PASS',
- 'actual': 'IMAGE',
- }, 'multiple-match-failure.html': {
- 'reftest_type': ['=='],
- 'report': 'REGRESSION',
- 'expected': 'PASS',
- 'actual': 'IMAGE',
- 'image_diff_percent': 1,
- }, 'multiple-mismatch-failure.html': {
- 'reftest_type': ['!='],
- 'report': 'REGRESSION',
- 'expected': 'PASS',
- 'actual': 'IMAGE',
- }, 'test.html': {
- 'reftest_type': ['=='],
- 'report': 'REGRESSION',
- 'expected': 'PASS',
- 'actual': 'IMAGE',
- 'image_diff_percent': None,
- }, 'unlistedtest.html': {
- 'report': 'MISSING',
- 'expected': 'PASS',
- 'actual': 'MISSING',
- 'is_missing_text': True,
- 'is_missing_image': True,
- },
- },
- },
- },
- }
- actual_dictionary = json.loads(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')[len('ADD_RESULTS('):-2])
-
- self.assertEqual(
- expected_dictionary['tests']['reftests']['foo']['unlistedtest.html'],
- actual_dictionary['tests']['reftests']['foo']['unlistedtest.html'],
- )
- self.assertEqual(expected_dictionary['num_regressions'], actual_dictionary['num_regressions'])
- self.assertEqual(expected_dictionary['num_flaky'], actual_dictionary['num_flaky'])
- self.assertEqual(expected_dictionary['num_missing'], actual_dictionary['num_missing'])
-
def test_additional_platform_directory(self):
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))
self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo']))
@@ -1044,36 +980,9 @@
self.assertEqual(3, len(by_type[DeviceType.from_string('iPad (5th generation)')]))
-class EndToEndTest(unittest.TestCase):
- def test_reftest_with_two_notrefs(self):
- # Test that we update expectations in place. If the expectation
- # is missing, update the expected generic location.
- host = MockHost()
- _, _, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host)
- file_list = host.filesystem.written_files.keys()
-
- json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
- json = parse_full_results(json_string)
- self.assertTrue("multiple-match-success.html" not in json["tests"]["reftests"]["foo"])
- self.assertTrue("multiple-mismatch-success.html" not in json["tests"]["reftests"]["foo"])
- self.assertTrue("multiple-both-success.html" not in json["tests"]["reftests"]["foo"])
- self.assertEqual(
- json["tests"]["reftests"]["foo"]["multiple-match-failure.html"],
- {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["=="], "image_diff_percent": 1, "report": "REGRESSION"},
- )
- self.assertEqual(
- json["tests"]["reftests"]["foo"]["multiple-mismatch-failure.html"],
- {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["!="], "report": "REGRESSION"},
- )
- self.assertEqual(
- json["tests"]["reftests"]["foo"]["multiple-both-failure.html"],
- {"expected": "PASS", "actual": "IMAGE", "reftest_type": sorted(["==", "!="]), "report": "REGRESSION"},
- )
-
-
class RebaselineTest(unittest.TestCase, StreamTestingMixin):
def assertBaselines(self, file_list, file, extensions, err):
- "assert that the file_list contains the baselines."""
+ """assert that the file_list contains the baselines."""
for ext in extensions:
baseline = file + "-expected" + ext
baseline_msg = 'Writing new expected result "%s"\n' % baseline
Modified: trunk/Tools/Scripts/webkitpy/port/base.py (270953 => 270954)
--- trunk/Tools/Scripts/webkitpy/port/base.py 2020-12-17 23:17:38 UTC (rev 270953)
+++ trunk/Tools/Scripts/webkitpy/port/base.py 2020-12-17 23:19:51 UTC (rev 270954)
@@ -130,7 +130,6 @@
if not hasattr(options, 'configuration') or not options.configuration:
self.set_option_default('configuration', self.default_configuration())
self._test_configuration = None
- self._reftest_list = {}
self._results_directory = None
self._root_was_set = hasattr(options, 'root') and options.root
self._jhbuild_wrapper = []
@@ -494,29 +493,6 @@
text = string_utils.decode(self._filesystem.read_binary_file(baseline_path), target_type=str)
return text.replace("\r\n", "\n")
- def _get_reftest_list(self, test_name):
- dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name))
- if dirname not in self._reftest_list:
- self._reftest_list[dirname] = Port._parse_reftest_list(self._filesystem, dirname)
- return self._reftest_list[dirname]
-
- @staticmethod
- def _parse_reftest_list(filesystem, test_dirpath):
- reftest_list_path = filesystem.join(test_dirpath, 'reftest.list')
- if not filesystem.isfile(reftest_list_path):
- return None
- reftest_list_file = filesystem.read_text_file(reftest_list_path)
-
- parsed_list = {}
- for line in reftest_list_file.split('\n'):
- line = re.sub('#.+$', '', line)
- split_line = line.split()
- if len(split_line) < 3:
- continue
- expectation_type, test_file, ref_file = split_line
- parsed_list.setdefault(filesystem.join(test_dirpath, test_file), []).append((expectation_type, filesystem.join(test_dirpath, ref_file)))
- return parsed_list
-
def reference_files(self, test_name, device_type=None):
"""Return a list of expectation (== or !=) and filename pairs"""
@@ -523,10 +499,6 @@
if self.get_option('treat_ref_tests_as_pixel_tests'):
return []
- result = self._get_reftest_list(test_name)
- if result:
- return result.get(self._filesystem.join(self.layout_tests_dir(), test_name), [])
-
result = []
suffixes = []
for part1 in ['', '-mismatch']:
Modified: trunk/Tools/Scripts/webkitpy/port/base_unittest.py (270953 => 270954)
--- trunk/Tools/Scripts/webkitpy/port/base_unittest.py 2020-12-17 23:17:38 UTC (rev 270953)
+++ trunk/Tools/Scripts/webkitpy/port/base_unittest.py 2020-12-17 23:19:51 UTC (rev 270954)
@@ -289,26 +289,6 @@
self.assertFalse(Port.is_reference_html_file(filesystem, '', 'foo-expected.php'))
self.assertFalse(Port.is_reference_html_file(filesystem, '', 'foo-expected.mht'))
- def test_parse_reftest_list(self):
- port = self.make_port(with_tests=True)
- port.host.filesystem.write_text_file(
- 'bar/reftest.list',
- "\n".join([
- "== test.html test-ref.html",
- "",
- "# some comment",
- "!= test-2.html test-notref.html # more comments",
- "== test-3.html test-ref.html",
- "== test-3.html test-ref2.html",
- "!= test-3.html test-notref.html",
- ]),
- )
-
- reftest_list = Port._parse_reftest_list(port.host.filesystem, 'bar')
- self.assertEqual(reftest_list, {'bar/test.html': [('==', 'bar/test-ref.html')],
- 'bar/test-2.html': [('!=', 'bar/test-notref.html')],
- 'bar/test-3.html': [('==', 'bar/test-ref.html'), ('==', 'bar/test-ref2.html'), ('!=', 'bar/test-notref.html')]})
-
def test_reference_files(self):
port = self.make_port(with_tests=True)
self.assertEqual(port.reference_files('passes/svgreftest.svg'), [('==', port.layout_tests_dir() + '/passes/svgreftest-expected.svg')])
Modified: trunk/Tools/Scripts/webkitpy/port/test.py (270953 => 270954)
--- trunk/Tools/Scripts/webkitpy/port/test.py 2020-12-17 23:17:38 UTC (rev 270953)
+++ trunk/Tools/Scripts/webkitpy/port/test.py 2020-12-17 23:19:51 UTC (rev 270954)
@@ -101,15 +101,16 @@
def __getitem__(self, item):
return self.tests[item]
+
#
# These numbers may need to be updated whenever we add or delete tests.
#
-TOTAL_TESTS = 77
+TOTAL_TESTS = 69
TOTAL_SKIPS = 9
-TOTAL_RETRIES = 15
+TOTAL_RETRIES = 11
-UNEXPECTED_PASSES = 7
-UNEXPECTED_FAILURES = 18
+UNEXPECTED_PASSES = 6
+UNEXPECTED_FAILURES = 14
def unit_test_list():
@@ -233,25 +234,7 @@
tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True)
tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True)
# FIXME: Add a reftest which crashes.
- tests.add('reftests/foo/test.html')
- tests.add('reftests/foo/test-ref.html')
- tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc')
- tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc')
- tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='abc', actual_image='abc')
- tests.add('reftests/foo/multiple-mismatch-failure.html', actual_checksum='abc', actual_image='abc')
- tests.add('reftests/foo/multiple-both-success.html', actual_checksum='abc', actual_image='abc')
- tests.add('reftests/foo/multiple-both-failure.html', actual_checksum='abc', actual_image='abc')
-
- tests.add('reftests/foo/matching-ref.html', actual_checksum='abc', actual_image='abc')
- tests.add('reftests/foo/mismatching-ref.html', actual_checksum='def', actual_image='def')
- tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi')
-
- # The following files shouldn't be treated as reftests
- tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True)
- tests.add('reftests/foo/reference/bar/common.html')
- tests.add('reftests/foo/reftest/bar/shared.html')
-
tests.add('websocket/tests/passes/text.html')
# For testing test are properly included from platform directories.
@@ -320,26 +303,6 @@
Bug(test) passes/skipped/skip.html [ Skip ]
""")
- filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/reftests/foo')
- filesystem.write_text_file(LAYOUT_TEST_DIR + '/reftests/foo/reftest.list', """
-== test.html test-ref.html
-
-== multiple-match-success.html mismatching-ref.html
-== multiple-match-success.html matching-ref.html
-== multiple-match-failure.html mismatching-ref.html
-== multiple-match-failure.html second-mismatching-ref.html
-!= multiple-mismatch-success.html mismatching-ref.html
-!= multiple-mismatch-success.html second-mismatching-ref.html
-!= multiple-mismatch-failure.html mismatching-ref.html
-!= multiple-mismatch-failure.html matching-ref.html
-== multiple-both-success.html matching-ref.html
-== multiple-both-success.html mismatching-ref.html
-!= multiple-both-success.html second-mismatching-ref.html
-== multiple-both-failure.html matching-ref.html
-!= multiple-both-failure.html second-mismatching-ref.html
-!= multiple-both-failure.html matching-ref.html
-""")
-
# FIXME: This test was only being ignored because of missing a leading '/'.
# Fixing the typo causes several tests to assert, so disabling the test entirely.
# Add in a file should be ignored by port.find_test_files().