Diff
Modified: trunk/LayoutTests/ChangeLog (287322 => 287323)
--- trunk/LayoutTests/ChangeLog 2021-12-21 18:31:16 UTC (rev 287322)
+++ trunk/LayoutTests/ChangeLog 2021-12-21 18:34:35 UTC (rev 287323)
@@ -1,3 +1,21 @@
+2021-12-21 Simon Fraser <[email protected]>
+
+ Allow a way to specify a custom pixel tolerance in a ref test
+ https://bugs.webkit.org/show_bug.cgi?id=149828
+
+ Reviewed by Martin Robinson.
+
+ Add some ref tests using <meta name=fuzzy> to test fuzzy-based passes and failures.
+
+ * fast/harness/image-diff/fuzz-2-25-expected.html: Added.
+ * fast/harness/image-diff/fuzz-2-25.html: Added.
+ * fast/harness/image-diff/fuzz-2-36-over-expected-mismatch.html: Added.
+ * fast/harness/image-diff/fuzz-2-36-over.html: Added.
+ * fast/harness/image-diff/fuzz-2-36-under-expected-mismatch.html: Added.
+ * fast/harness/image-diff/fuzz-2-36-under.html: Added.
+ * fast/harness/image-diff/zero-fuzz-expected.html: Added.
+ * fast/harness/image-diff/zero-fuzz.html: Added.
+
2021-12-21 Sam Weinig <[email protected]>
Add support for premultiplied alpha interpolated gradients and defaulted off option to use them for CSS Gradients
Modified: trunk/LayoutTests/TestExpectations (287322 => 287323)
--- trunk/LayoutTests/TestExpectations 2021-12-21 18:31:16 UTC (rev 287322)
+++ trunk/LayoutTests/TestExpectations 2021-12-21 18:34:35 UTC (rev 287323)
@@ -3740,6 +3740,8 @@
webkit.org/b/208052 imported/w3c/web-platform-tests/intersection-observer/v2/cross-origin-occlusion.sub.html [ Skip ]
# wpt css-backgrounds failures
+webkit.org/b/232717 imported/w3c/web-platform-tests/css/css-backgrounds/border-radius-clip-001.html [ ImageOnlyFailure ]
+
webkit.org/b/206753 imported/w3c/web-platform-tests/css/css-backgrounds/background-attachment-local/attachment-local-clipping-color-4.html [ ImageOnlyFailure ]
webkit.org/b/206753 imported/w3c/web-platform-tests/css/css-backgrounds/background-attachment-local/attachment-local-clipping-color-6.html [ ImageOnlyFailure ]
webkit.org/b/206753 imported/w3c/web-platform-tests/css/css-backgrounds/background-attachment-local/attachment-local-clipping-image-4.html [ ImageOnlyFailure ]
Added: trunk/LayoutTests/fast/harness/image-diff/fuzz-2-25-expected.html (0 => 287323)
--- trunk/LayoutTests/fast/harness/image-diff/fuzz-2-25-expected.html (rev 0)
+++ trunk/LayoutTests/fast/harness/image-diff/fuzz-2-25-expected.html 2021-12-21 18:34:35 UTC (rev 287323)
@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <style>
+ .box {
+ position: relative;
+ width: 100px;
+ height: 100px;
+ background-color: rgb(0, 128, 0);
+ }
+ </style>
+</head>
+<body>
+ <div class="box"></div>
+</body>
+</html>
Added: trunk/LayoutTests/fast/harness/image-diff/fuzz-2-25.html (0 => 287323)
--- trunk/LayoutTests/fast/harness/image-diff/fuzz-2-25.html (rev 0)
+++ trunk/LayoutTests/fast/harness/image-diff/fuzz-2-25.html 2021-12-21 18:34:35 UTC (rev 287323)
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <meta name="fuzzy" content="maxDifference=2;totalPixels=25">
+ <style>
+ .box {
+ position: relative;
+ width: 100px;
+ height: 100px;
+ background-color: rgb(0, 128, 0);
+ }
+
+ .mismatch-box {
+ position: absolute;
+ margin: 10px;
+ width: 5px;
+ height: 5px;
+ background-color: rgb(0, 126, 0);
+ }
+ </style>
+</head>
+<body>
+ <div class="box">
+ <div class="mismatch-box"></div>
+ </div>
+</body>
+</html>
Added: trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-over-expected-mismatch.html (0 => 287323)
--- trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-over-expected-mismatch.html (rev 0)
+++ trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-over-expected-mismatch.html 2021-12-21 18:34:35 UTC (rev 287323)
@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <style>
+ .box {
+ position: relative;
+ width: 100px;
+ height: 100px;
+ background-color: rgb(0, 128, 0);
+ }
+ </style>
+</head>
+<body>
+ <div class="box"></div>
+</body>
+</html>
Added: trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-over.html (0 => 287323)
--- trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-over.html (rev 0)
+++ trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-over.html 2021-12-21 18:34:35 UTC (rev 287323)
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <meta name="fuzzy" content="maxDifference=2;totalPixels=36">
+ <style>
+ .box {
+ position: relative;
+ width: 100px;
+ height: 100px;
+ background-color: rgb(0, 128, 0);
+ }
+
+ .mismatch-box {
+ position: absolute;
+ margin: 10px;
+ width: 6px;
+ height: 7px;
+ background-color: rgb(0, 126, 0);
+ }
+ </style>
+</head>
+<body>
+ <div class="box">
+ <div class="mismatch-box"></div>
+ </div>
+</body>
+</html>
Added: trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-under-expected-mismatch.html (0 => 287323)
--- trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-under-expected-mismatch.html (rev 0)
+++ trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-under-expected-mismatch.html 2021-12-21 18:34:35 UTC (rev 287323)
@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <style>
+ .box {
+ position: relative;
+ width: 100px;
+ height: 100px;
+ background-color: rgb(0, 128, 0);
+ }
+ </style>
+</head>
+<body>
+ <div class="box"></div>
+</body>
+</html>
Added: trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-under.html (0 => 287323)
--- trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-under.html (rev 0)
+++ trunk/LayoutTests/fast/harness/image-diff/fuzz-2-36-under.html 2021-12-21 18:34:35 UTC (rev 287323)
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <meta name="fuzzy" content="maxDifference=2;totalPixels=36">
+ <style>
+ .box {
+ position: relative;
+ width: 100px;
+ height: 100px;
+ background-color: rgb(0, 128, 0);
+ }
+
+ .mismatch-box {
+ position: absolute;
+ margin: 10px;
+ width: 6px;
+ height: 6px;
+ background-color: rgb(0, 127, 0);
+ }
+ </style>
+</head>
+<body>
+ <div class="box">
+ <div class="mismatch-box"></div>
+ </div>
+</body>
+</html>
Added: trunk/LayoutTests/fast/harness/image-diff/zero-fuzz-expected.html (0 => 287323)
--- trunk/LayoutTests/fast/harness/image-diff/zero-fuzz-expected.html (rev 0)
+++ trunk/LayoutTests/fast/harness/image-diff/zero-fuzz-expected.html 2021-12-21 18:34:35 UTC (rev 287323)
@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <style>
+ .box {
+ width: 100px;
+ height: 100px;
+ background-color: rgb(0, 128, 0);
+ }
+ </style>
+</head>
+<body>
+ <div class="box"></div>
+</body>
+</html>
Added: trunk/LayoutTests/fast/harness/image-diff/zero-fuzz.html (0 => 287323)
--- trunk/LayoutTests/fast/harness/image-diff/zero-fuzz.html (rev 0)
+++ trunk/LayoutTests/fast/harness/image-diff/zero-fuzz.html 2021-12-21 18:34:35 UTC (rev 287323)
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <meta name=fuzzy content="0;0">
+ <style>
+ .box {
+ width: 100px;
+ height: 100px;
+ background-color: rgb(0, 128, 0);
+ }
+
+ </style>
+</head>
+<body>
+ <div class="box"></div>
+</body>
+</html>
Modified: trunk/LayoutTests/platform/ios-wk2/TestExpectations (287322 => 287323)
--- trunk/LayoutTests/platform/ios-wk2/TestExpectations 2021-12-21 18:31:16 UTC (rev 287322)
+++ trunk/LayoutTests/platform/ios-wk2/TestExpectations 2021-12-21 18:34:35 UTC (rev 287323)
@@ -1112,6 +1112,8 @@
webkit.org/b/157589 fast/text-autosizing/ios/text-autosizing-after-back.html [ Pass Timeout ]
+webkit.org/b/232525 fast/harness/image-diff/fuzz-2-25.html [ ImageOnlyFailure ]
+
# started to fail after new snapshotting in r202532
webkit.org/b/150942 animations/multiple-backgrounds.html [ Pass ImageOnlyFailure ]
compositing/overlap-blending/reflection-opacity-huge.html [ ImageOnlyFailure ]
Modified: trunk/Tools/ChangeLog (287322 => 287323)
--- trunk/Tools/ChangeLog 2021-12-21 18:31:16 UTC (rev 287322)
+++ trunk/Tools/ChangeLog 2021-12-21 18:34:35 UTC (rev 287323)
@@ -1,3 +1,22 @@
+2021-12-21 Simon Fraser <[email protected]>
+
+ Allow a way to specify a custom pixel tolerance in a ref test
+ https://bugs.webkit.org/show_bug.cgi?id=149828
+
+ Reviewed by Martin Robinson.
+
+ When determining if a ref test failed, look for <meta name=fuzzy> in the test file,
+ and if present, compare the actual pixel differences with the range allowed by the
+ fuzzy metadata. This may cause a failing reference to match, or a passing reference
+ (based on ImageDiff internal tolerance) to fail.
+
+ In the case of mismatch ref tests, the fuzziness is taken into account to determine
+ if the mismatch still occurs. If not, then the mismatch test fails.
+
+ * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
+ (SingleTestRunner._fuzzy_tolerance_for_reference):
+ (SingleTestRunner._compare_output_with_reference):
+
2021-12-21 Alex Christensen <[email protected]>
ASSERT NOT REACHED under WebKit::Daemon::ConnectionToMachService seen with TestWebKitAPI.WebPushD.BasicCommunication and PermissionManagement
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py (287322 => 287323)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2021-12-21 18:31:16 UTC (rev 287322)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2021-12-21 18:34:35 UTC (rev 287323)
@@ -370,7 +370,7 @@
test_parser = TestParser({'all': True}, filename=test_full_path, host=self._port.host)
fuzzy = test_parser.fuzzy_metadata()
if not fuzzy:
- return {'maxDifference': [0, 0], 'totalPixels': [0, 0]}
+ return None
reference_relative_path = self._relative_reference_path(test_full_path, reference_full_path)
tolerance = [[0, 0], [0, 0]]
@@ -403,6 +403,7 @@
if failures:
# Don't continue any more if we already have crash or timeout.
return TestResult(self._test_input, failures, total_test_time, has_stderr)
+
failures.extend(self._handle_error(reference_driver_output, reference_filename=reference_filename))
if failures:
return TestResult(self._test_input, failures, total_test_time, has_stderr, pid=actual_driver_output.pid)
@@ -410,14 +411,27 @@
if not reference_driver_output.image_hash and not actual_driver_output.image_hash:
failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename))
elif mismatch:
- # Calling image_hash is considered unnecessary for expected mismatch ref tests.
- if reference_driver_output.image_hash == actual_driver_output.image_hash:
- failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename))
+ fuzzy_tolerance = self._fuzzy_tolerance_for_reference(reference_filename)
+ if fuzzy_tolerance:
+ diff_result = self._port.diff_image(reference_driver_output.image, actual_driver_output.image, tolerance=0)
+ match_within_tolerance = self._test_passes_fuzzy_matching(fuzzy_tolerance, diff_result.fuzzy_data)
+ _log.debug('{} expected mismatch: allowed fuzziness {}, actual difference {}: matched {}'.format(self._test_name, fuzzy_tolerance, diff_result.fuzzy_data, match_within_tolerance))
+ if match_within_tolerance:
+ # FIXME: we could store and show the diff image for mismatch failures.
+ failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename))
+ else:
+ # Calling image_hash is considered unnecessary for expected mismatch ref tests.
+ if reference_driver_output.image_hash == actual_driver_output.image_hash:
+ failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename))
+
elif reference_driver_output.image_hash != actual_driver_output.image_hash:
# ImageDiff has a hard coded color distance threshold even though tolerance=0 is specified.
diff_result = self._port.diff_image(reference_driver_output.image, actual_driver_output.image, tolerance=0)
- # FIXME: use the result of _fuzzy_tolerance_for_reference to allow for pass or fail based on fuzzy matching. webkit.org/b/149828
+ fuzzy_tolerance = self._fuzzy_tolerance_for_reference(reference_filename)
+ if fuzzy_tolerance:
+ diff_result.passed = self._test_passes_fuzzy_matching(fuzzy_tolerance, diff_result.fuzzy_data)
+ _log.debug('{} allowed fuzziness {}, actual difference {}: pass {}'.format(self._test_name, fuzzy_tolerance, diff_result.fuzzy_data, diff_result.passed))
if not diff_result.passed:
failures.append(test_failures.FailureReftestMismatch(reference_filename, diff_result))
