Title: [270954] trunk/Tools
Revision
270954
Author
[email protected]
Date
2020-12-17 15:19:51 -0800 (Thu, 17 Dec 2020)

Log Message

Stop parsing reftest.list files
https://bugs.webkit.org/show_bug.cgi?id=203783

Patch by Sam Sneddon <[email protected]> on 2020-12-17
Reviewed by Jonathan Bedard.

This has only been used by the webkitpy integration tests in recent years. Needless to say,
this is Not Useful.

We do still have one reftest.list file,
media/track/opera/track/webvtt/rendering/reftest/reftest.list, but as this is in a reftest
directory we don't run any of the tests in it (and never have). See the above bug for more
detail about that history.

* Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
(RebaselineTest.assertBaselines): docstring fix
(RunTest.test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist): Deleted.
(EndToEndTest): Deleted.
(EndToEndTest.test_reftest_with_two_notrefs): Deleted.
* Scripts/webkitpy/port/base.py:
(Port.__init__): remove Port._reftest_list
(Port.reference_files): don't use reftest.list as source of truth
(Port._get_reftest_list): Deleted.
(Port._parse_reftest_list): Deleted.
* Scripts/webkitpy/port/base_unittest.py:
(PortTest.test_parse_reftest_list): Deleted.
* Scripts/webkitpy/port/test.py:
(unit_test_list): remove tests used to test reftest.list

Modified Paths

Diff

Modified: trunk/Tools/ChangeLog (270953 => 270954)


--- trunk/Tools/ChangeLog	2020-12-17 23:17:38 UTC (rev 270953)
+++ trunk/Tools/ChangeLog	2020-12-17 23:19:51 UTC (rev 270954)
@@ -1,3 +1,33 @@
+2020-12-17  Sam Sneddon  <[email protected]>
+
+        Stop parsing reftest.list files
+        https://bugs.webkit.org/show_bug.cgi?id=203783
+
+        Reviewed by Jonathan Bedard.
+
+        This has only been used by the webkitpy integration tests in recent years. Needless to say,
+        this is Not Useful.
+
+        We do still have one reftest.list file,
+        media/track/opera/track/webvtt/rendering/reftest/reftest.list, but as this is in a reftest
+        directory we don't run any of the tests in it (and never have). See the above bug for more
+        detail about that history.
+
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+        (RebaselineTest.assertBaselines): docstring fix
+        (RunTest.test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist): Deleted.
+        (EndToEndTest): Deleted.
+        (EndToEndTest.test_reftest_with_two_notrefs): Deleted.
+        * Scripts/webkitpy/port/base.py:
+        (Port.__init__): remove Port._reftest_list
+        (Port.reference_files): don't use reftest.list as source of truth
+        (Port._get_reftest_list): Deleted.
+        (Port._parse_reftest_list): Deleted.
+        * Scripts/webkitpy/port/base_unittest.py:
+        (PortTest.test_parse_reftest_list): Deleted.
+        * Scripts/webkitpy/port/test.py:
+        (unit_test_list): remove tests used to test reftest.list
+
 2020-12-17  Jonathan Bedard  <[email protected]>
 
         [run-webkit-tests] Replace FailureNotTested with FailureNoOutput

Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (270953 => 270954)


--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py	2020-12-17 23:17:38 UTC (rev 270953)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py	2020-12-17 23:19:51 UTC (rev 270954)
@@ -807,70 +807,6 @@
         tests_run = get_test_results(['passes/mismatch.html'])
         self.assertEqual(tests_run[0].references, ['passes/mismatch-expected-mismatch.html'])
 
-    def test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist(self):
-        host = MockHost()
-        _, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host)
-        expected_dictionary = {
-            'version': 4,
-            'fixable': 5,
-            'skipped': 0,
-            'num_passes': 3,
-            'num_flaky': 0,
-            'num_missing': 1,
-            'num_regressions': 4,
-            'uses_expectations_file': True,
-            'interrupted': False,
-            'layout_tests_dir': '/test.checkout/LayoutTests',
-            'has_pretty_patch': False,
-            'pixel_tests_enabled': True,
-            'other_crashes': {},
-            'date': '10:27AM on December 13, 2019',
-            'tests': {
-                'reftests': {
-                    'foo': {
-                        'multiple-both-failure.html': {
-                            'reftest_type': ['!=', '=='],
-                            'report': 'REGRESSION',
-                            'expected': 'PASS',
-                            'actual': 'IMAGE',
-                        }, 'multiple-match-failure.html': {
-                            'reftest_type': ['=='],
-                            'report': 'REGRESSION',
-                            'expected': 'PASS',
-                            'actual': 'IMAGE',
-                            'image_diff_percent': 1,
-                        }, 'multiple-mismatch-failure.html': {
-                            'reftest_type': ['!='],
-                            'report': 'REGRESSION',
-                            'expected': 'PASS',
-                            'actual': 'IMAGE',
-                        }, 'test.html': {
-                            'reftest_type': ['=='],
-                            'report': 'REGRESSION',
-                            'expected': 'PASS',
-                            'actual': 'IMAGE',
-                            'image_diff_percent': None,
-                        }, 'unlistedtest.html': {
-                            'report': 'MISSING',
-                            'expected': 'PASS',
-                            'actual': 'MISSING',
-                            'is_missing_text': True,
-                            'is_missing_image': True,
-                        },
-                    },
-                },
-            },
-        }
-        actual_dictionary = json.loads(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')[len('ADD_RESULTS('):-2])
-
-        self.assertEqual(
-            expected_dictionary['tests']['reftests']['foo']['unlistedtest.html'],
-            actual_dictionary['tests']['reftests']['foo']['unlistedtest.html'],
-        )
-        self.assertEqual(expected_dictionary['num_regressions'], actual_dictionary['num_regressions'])
-        self.assertEqual(expected_dictionary['num_flaky'], actual_dictionary['num_flaky'])
-        self.assertEqual(expected_dictionary['num_missing'], actual_dictionary['num_missing'])
-
     def test_additional_platform_directory(self):
         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))
         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo']))
@@ -1044,36 +980,9 @@
         self.assertEqual(3, len(by_type[DeviceType.from_string('iPad (5th generation)')]))
 
 
-class EndToEndTest(unittest.TestCase):
-    def test_reftest_with_two_notrefs(self):
-        # Test that we update expectations in place. If the expectation
-        # is missing, update the expected generic location.
-        host = MockHost()
-        _, _, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host)
-        file_list = host.filesystem.written_files.keys()
-
-        json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
-        json = parse_full_results(json_string)
-        self.assertTrue("multiple-match-success.html" not in json["tests"]["reftests"]["foo"])
-        self.assertTrue("multiple-mismatch-success.html" not in json["tests"]["reftests"]["foo"])
-        self.assertTrue("multiple-both-success.html" not in json["tests"]["reftests"]["foo"])
-        self.assertEqual(
-            json["tests"]["reftests"]["foo"]["multiple-match-failure.html"],
-            {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["=="], "image_diff_percent": 1, "report": "REGRESSION"},
-        )
-        self.assertEqual(
-            json["tests"]["reftests"]["foo"]["multiple-mismatch-failure.html"],
-            {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["!="], "report": "REGRESSION"},
-        )
-        self.assertEqual(
-            json["tests"]["reftests"]["foo"]["multiple-both-failure.html"],
-            {"expected": "PASS", "actual": "IMAGE", "reftest_type": sorted(["==", "!="]), "report": "REGRESSION"},
-        )
-
-
 class RebaselineTest(unittest.TestCase, StreamTestingMixin):
     def assertBaselines(self, file_list, file, extensions, err):
-        "assert that the file_list contains the baselines."""
+        """assert that the file_list contains the baselines."""
         for ext in extensions:
             baseline = file + "-expected" + ext
             baseline_msg = 'Writing new expected result "%s"\n' % baseline

Modified: trunk/Tools/Scripts/webkitpy/port/base.py (270953 => 270954)


--- trunk/Tools/Scripts/webkitpy/port/base.py	2020-12-17 23:17:38 UTC (rev 270953)
+++ trunk/Tools/Scripts/webkitpy/port/base.py	2020-12-17 23:19:51 UTC (rev 270954)
@@ -130,7 +130,6 @@
         if not hasattr(options, 'configuration') or not options.configuration:
             self.set_option_default('configuration', self.default_configuration())
         self._test_configuration = None
-        self._reftest_list = {}
         self._results_directory = None
         self._root_was_set = hasattr(options, 'root') and options.root
         self._jhbuild_wrapper = []
@@ -494,29 +493,6 @@
         text = string_utils.decode(self._filesystem.read_binary_file(baseline_path), target_type=str)
         return text.replace("\r\n", "\n")
 
-    def _get_reftest_list(self, test_name):
-        dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name))
-        if dirname not in self._reftest_list:
-            self._reftest_list[dirname] = Port._parse_reftest_list(self._filesystem, dirname)
-        return self._reftest_list[dirname]
-
-    @staticmethod
-    def _parse_reftest_list(filesystem, test_dirpath):
-        reftest_list_path = filesystem.join(test_dirpath, 'reftest.list')
-        if not filesystem.isfile(reftest_list_path):
-            return None
-        reftest_list_file = filesystem.read_text_file(reftest_list_path)
-
-        parsed_list = {}
-        for line in reftest_list_file.split('\n'):
-            line = re.sub('#.+$', '', line)
-            split_line = line.split()
-            if len(split_line) < 3:
-                continue
-            expectation_type, test_file, ref_file = split_line
-            parsed_list.setdefault(filesystem.join(test_dirpath, test_file), []).append((expectation_type, filesystem.join(test_dirpath, ref_file)))
-        return parsed_list
-
     def reference_files(self, test_name, device_type=None):
         """Return a list of expectation (== or !=) and filename pairs"""
 
@@ -523,10 +499,6 @@
         if self.get_option('treat_ref_tests_as_pixel_tests'):
             return []
 
-        result = self._get_reftest_list(test_name)
-        if result:
-            return result.get(self._filesystem.join(self.layout_tests_dir(), test_name), [])
-
         result = []
         suffixes = []
         for part1 in ['', '-mismatch']:

Modified: trunk/Tools/Scripts/webkitpy/port/base_unittest.py (270953 => 270954)


--- trunk/Tools/Scripts/webkitpy/port/base_unittest.py	2020-12-17 23:17:38 UTC (rev 270953)
+++ trunk/Tools/Scripts/webkitpy/port/base_unittest.py	2020-12-17 23:19:51 UTC (rev 270954)
@@ -289,26 +289,6 @@
         self.assertFalse(Port.is_reference_html_file(filesystem, '', 'foo-expected.php'))
         self.assertFalse(Port.is_reference_html_file(filesystem, '', 'foo-expected.mht'))
 
-    def test_parse_reftest_list(self):
-        port = self.make_port(with_tests=True)
-        port.host.filesystem.write_text_file(
-            'bar/reftest.list',
-            "\n".join([
-                "== test.html test-ref.html",
-                "",
-                "# some comment",
-                "!= test-2.html test-notref.html # more comments",
-                "== test-3.html test-ref.html",
-                "== test-3.html test-ref2.html",
-                "!= test-3.html test-notref.html",
-            ]),
-        )
-
-        reftest_list = Port._parse_reftest_list(port.host.filesystem, 'bar')
-        self.assertEqual(reftest_list, {'bar/test.html': [('==', 'bar/test-ref.html')],
-            'bar/test-2.html': [('!=', 'bar/test-notref.html')],
-            'bar/test-3.html': [('==', 'bar/test-ref.html'), ('==', 'bar/test-ref2.html'), ('!=', 'bar/test-notref.html')]})
-
     def test_reference_files(self):
         port = self.make_port(with_tests=True)
         self.assertEqual(port.reference_files('passes/svgreftest.svg'), [('==', port.layout_tests_dir() + '/passes/svgreftest-expected.svg')])

Modified: trunk/Tools/Scripts/webkitpy/port/test.py (270953 => 270954)


--- trunk/Tools/Scripts/webkitpy/port/test.py	2020-12-17 23:17:38 UTC (rev 270953)
+++ trunk/Tools/Scripts/webkitpy/port/test.py	2020-12-17 23:19:51 UTC (rev 270954)
@@ -101,15 +101,16 @@
     def __getitem__(self, item):
         return self.tests[item]
 
+
 #
 # These numbers may need to be updated whenever we add or delete tests.
 #
-TOTAL_TESTS = 77
+TOTAL_TESTS = 69
 TOTAL_SKIPS = 9
-TOTAL_RETRIES = 15
+TOTAL_RETRIES = 11
 
-UNEXPECTED_PASSES = 7
-UNEXPECTED_FAILURES = 18
+UNEXPECTED_PASSES = 6
+UNEXPECTED_FAILURES = 14
 
 
 def unit_test_list():
@@ -233,25 +234,7 @@
     tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True)
     tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True)
     # FIXME: Add a reftest which crashes.
-    tests.add('reftests/foo/test.html')
-    tests.add('reftests/foo/test-ref.html')
 
-    tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc')
-    tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc')
-    tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='abc', actual_image='abc')
-    tests.add('reftests/foo/multiple-mismatch-failure.html', actual_checksum='abc', actual_image='abc')
-    tests.add('reftests/foo/multiple-both-success.html', actual_checksum='abc', actual_image='abc')
-    tests.add('reftests/foo/multiple-both-failure.html', actual_checksum='abc', actual_image='abc')
-
-    tests.add('reftests/foo/matching-ref.html', actual_checksum='abc', actual_image='abc')
-    tests.add('reftests/foo/mismatching-ref.html', actual_checksum='def', actual_image='def')
-    tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi')
-
-    # The following files shouldn't be treated as reftests
-    tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True)
-    tests.add('reftests/foo/reference/bar/common.html')
-    tests.add('reftests/foo/reftest/bar/shared.html')
-
     tests.add('websocket/tests/passes/text.html')
 
     # For testing test are properly included from platform directories.
@@ -320,26 +303,6 @@
 Bug(test) passes/skipped/skip.html [ Skip ]
 """)
 
-    filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/reftests/foo')
-    filesystem.write_text_file(LAYOUT_TEST_DIR + '/reftests/foo/reftest.list', """
-== test.html test-ref.html
-
-== multiple-match-success.html mismatching-ref.html
-== multiple-match-success.html matching-ref.html
-== multiple-match-failure.html mismatching-ref.html
-== multiple-match-failure.html second-mismatching-ref.html
-!= multiple-mismatch-success.html mismatching-ref.html
-!= multiple-mismatch-success.html second-mismatching-ref.html
-!= multiple-mismatch-failure.html mismatching-ref.html
-!= multiple-mismatch-failure.html matching-ref.html
-== multiple-both-success.html matching-ref.html
-== multiple-both-success.html mismatching-ref.html
-!= multiple-both-success.html second-mismatching-ref.html
-== multiple-both-failure.html matching-ref.html
-!= multiple-both-failure.html second-mismatching-ref.html
-!= multiple-both-failure.html matching-ref.html
-""")
-
     # FIXME: This test was only being ignored because of missing a leading '/'.
     # Fixing the typo causes several tests to assert, so disabling the test entirely.
     # Add in a file should be ignored by port.find_test_files().
_______________________________________________
webkit-changes mailing list
[email protected]
https://lists.webkit.org/mailman/listinfo/webkit-changes

Reply via email to