Diff
Modified: trunk/LayoutTests/ChangeLog (279348 => 279349)
--- trunk/LayoutTests/ChangeLog 2021-06-28 21:24:48 UTC (rev 279348)
+++ trunk/LayoutTests/ChangeLog 2021-06-28 21:44:51 UTC (rev 279349)
@@ -1,3 +1,28 @@
+2021-06-28 Wenson Hsieh <[email protected]>
+
+ Live Text selections inside images is misaligned when "object-fit" is not "fill"
+ https://bugs.webkit.org/show_bug.cgi?id=227453
+
+ Reviewed by Tim Horton.
+
+ Add a couple of new tests, and do some minor cleanup in an existing test.
+
+ * fast/images/text-recognition/image-overlay-object-fit-change-expected.txt: Added.
+ * fast/images/text-recognition/image-overlay-object-fit-change.html: Added.
+
+ Add a layout test to verify that dynamically changing CSS "object-fit" values for an image with Live Text causes
+ the dimensions of the Live Text to update as well.
+
+ * fast/images/text-recognition/image-overlay-object-fit-expected.txt: Added.
+ * fast/images/text-recognition/image-overlay-object-fit.html: Added.
+
+ Add a layout test to verify that Live Text is injected correctly when using all 5 values of "object-fit".
+
+ * fast/images/text-recognition/image-overlay-size-change.html:
+
+ Clean up this layout test a bit: remove an unnecessarily included script file, add a missing `head` tag and
+ don't try to inject an image overlay 10 times.
+
2021-06-28 Amir Mark Jr <[email protected]>
[Catalina WK2 Debug/ iOS 14 Debug] fast/css-custom-paint/out-of-memory-while-adding-worklet-module.html is a flaky timeout
Added: trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit-change-expected.txt (0 => 279349)
--- trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit-change-expected.txt (rev 0)
+++ trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit-change-expected.txt 2021-06-28 21:44:51 UTC (rev 279349)
@@ -0,0 +1,8 @@
+PASS width is 50
+PASS height is 25
+PASS width became 100
+PASS height is 50
+PASS successfullyParsed is true
+
+TEST COMPLETE
+
Added: trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit-change.html (0 => 279349)
--- trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit-change.html (rev 0)
+++ trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit-change.html 2021-06-28 21:44:51 UTC (rev 279349)
@@ -0,0 +1,61 @@
+<!DOCTYPE html>
+<html>
+<head>
+<script src=""
+<style>
+img {
+ width: 100px;
+ height: 200px;
+}
+</style>
+</head>
+<body>
+<img src=""
+<script>
+jsTestIsAsync = true;
+let image = document.querySelector("img");
+width = 0;
+height = 0;
+
+function updateImageOverlayTextDimensions() {
+ let textBoundingRect = internals.shadowRoot(image).querySelector(".image-overlay-text").getBoundingClientRect();
+ width = textBoundingRect.width;
+ height = textBoundingRect.height;
+}
+
+addEventListener("load", async () => {
+ image.style.objectFit = "contain";
+ internals.installImageOverlay(image, [
+ {
+ topLeft : new DOMPointReadOnly(0, 0),
+ topRight : new DOMPointReadOnly(0.5, 0),
+ bottomRight : new DOMPointReadOnly(0.5, 0.25),
+ bottomLeft : new DOMPointReadOnly(0, 0.25),
+ children: [
+ {
+ text : "foo",
+ topLeft : new DOMPointReadOnly(0, 0),
+ topRight : new DOMPointReadOnly(0.5, 0),
+ bottomRight : new DOMPointReadOnly(0.5, 0.25),
+ bottomLeft : new DOMPointReadOnly(0, 0.25),
+ }
+ ]
+ }
+ ]);
+
+ updateImageOverlayTextDimensions();
+
+ shouldBe("width", "50");
+ shouldBe("height", "25");
+
+ image.style.objectFit = "cover";
+ setInterval(updateImageOverlayTextDimensions, 10);
+
+ await shouldBecomeEqual("width", "100");
+ shouldBe("height", "50");
+
+ finishJSTest();
+});
+</script>
+</body>
+</html>
\ No newline at end of file
Added: trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit-expected.txt (0 => 279349)
--- trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit-expected.txt (rev 0)
+++ trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit-expected.txt 2021-06-28 21:44:51 UTC (rev 279349)
@@ -0,0 +1,10 @@
+
+PASS textRects['fill'] is [0, 0, 100, 25]
+PASS textRects['contain'] is [50, 0, 50, 25]
+PASS textRects['cover'] is [0, -50, 100, 50]
+PASS textRects['none'] is [-100, -150, 200, 100]
+PASS textRects['scale-down'] is [50, 0, 50, 25]
+PASS successfullyParsed is true
+
+TEST COMPLETE
+
Added: trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit.html (0 => 279349)
--- trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit.html (rev 0)
+++ trunk/LayoutTests/fast/images/text-recognition/image-overlay-object-fit.html 2021-06-28 21:44:51 UTC (rev 279349)
@@ -0,0 +1,64 @@
+<!DOCTYPE html>
+<html>
+<head>
+<script src=""
+<style>
+body, html {
+ margin: 0;
+}
+img {
+ width: 200px;
+ height: 100px;
+ position: absolute;
+ top: 0;
+ left: 0;
+}
+</style>
+</head>
+<body>
+<img style="object-fit: fill;" src=""
+<img style="object-fit: contain;" src=""
+<img style="object-fit: cover;" src=""
+<img style="object-fit: none;" src=""
+<img style="object-fit: scale-down;" src=""
+<pre id="console"></pre>
+<script>
+function imageOverlayTextRect(image) {
+ let bounds = internals.shadowRoot(image).querySelector(".image-overlay-text").getBoundingClientRect();
+ return [ Math.round(bounds.left), Math.round(bounds.top), Math.round(bounds.width), Math.round(bounds.height) ];
+}
+
+addEventListener("load", async () => {
+ textRects = { };
+
+ document.querySelectorAll("img").forEach(image => {
+ internals.installImageOverlay(image, [
+ {
+ topLeft : new DOMPointReadOnly(0, 0),
+ topRight : new DOMPointReadOnly(0.5, 0),
+ bottomRight : new DOMPointReadOnly(0.5, 0.25),
+ bottomLeft : new DOMPointReadOnly(0, 0.25),
+ children: [
+ {
+ text : "foo",
+ topLeft : new DOMPointReadOnly(0, 0),
+ topRight : new DOMPointReadOnly(0.5, 0),
+ bottomRight : new DOMPointReadOnly(0.5, 0.25),
+ bottomLeft : new DOMPointReadOnly(0, 0.25),
+ }
+ ]
+ }
+ ]);
+
+ textRects[getComputedStyle(image).objectFit] = imageOverlayTextRect(image);
+ });
+
+ shouldBe("textRects['fill']", "[0, 0, 100, 25]");
+ shouldBe("textRects['contain']", "[50, 0, 50, 25]");
+ shouldBe("textRects['cover']", "[0, -50, 100, 50]");
+ shouldBe("textRects['none']", "[-100, -150, 200, 100]");
+ shouldBe("textRects['scale-down']", "[50, 0, 50, 25]");
+});
+</script>
+</body>
+</html>
\ No newline at end of file
Modified: trunk/LayoutTests/fast/images/text-recognition/image-overlay-size-change.html (279348 => 279349)
--- trunk/LayoutTests/fast/images/text-recognition/image-overlay-size-change.html 2021-06-28 21:24:48 UTC (rev 279348)
+++ trunk/LayoutTests/fast/images/text-recognition/image-overlay-size-change.html 2021-06-28 21:44:51 UTC (rev 279349)
@@ -1,7 +1,8 @@
<!DOCTYPE html>
<html>
+<head>
<script src=""
-<script src=""
+</head>
<body>
<img src=""
<script>
@@ -17,25 +18,23 @@
}
addEventListener("load", async () => {
- for (let i = 0; i < 10; ++i) {
- internals.installImageOverlay(image, [
- {
- topLeft : new DOMPointReadOnly(0.5, 0.5),
- topRight : new DOMPointReadOnly(1, 0.5),
- bottomRight : new DOMPointReadOnly(1, 1),
- bottomLeft : new DOMPointReadOnly(0.5, 1),
- children: [
- {
- text : "Hello",
- topLeft : new DOMPointReadOnly(0.5, 0.5),
- topRight : new DOMPointReadOnly(1, 0.5),
- bottomRight : new DOMPointReadOnly(1, 1),
- bottomLeft : new DOMPointReadOnly(0.5, 1),
- }
- ],
- }
- ]);
- }
+ internals.installImageOverlay(image, [
+ {
+ topLeft : new DOMPointReadOnly(0.5, 0.5),
+ topRight : new DOMPointReadOnly(1, 0.5),
+ bottomRight : new DOMPointReadOnly(1, 1),
+ bottomLeft : new DOMPointReadOnly(0.5, 1),
+ children: [
+ {
+ text : "Hello",
+ topLeft : new DOMPointReadOnly(0.5, 0.5),
+ topRight : new DOMPointReadOnly(1, 0.5),
+ bottomRight : new DOMPointReadOnly(1, 1),
+ bottomLeft : new DOMPointReadOnly(0.5, 1),
+ }
+ ],
+ }
+ ]);
updateImageOverlayTextDimensions();
Modified: trunk/Source/WebCore/ChangeLog (279348 => 279349)
--- trunk/Source/WebCore/ChangeLog 2021-06-28 21:24:48 UTC (rev 279348)
+++ trunk/Source/WebCore/ChangeLog 2021-06-28 21:44:51 UTC (rev 279349)
@@ -1,3 +1,35 @@
+2021-06-28 Wenson Hsieh <[email protected]>
+
+ Live Text selections inside images is misaligned when "object-fit" is not "fill"
+ https://bugs.webkit.org/show_bug.cgi?id=227453
+
+ Reviewed by Tim Horton.
+
+ We currently use the bounds of the image overlay host element when injecting transformed Live Text into image
+ element UA shadow roots. However, this causes text to lay out in the wrong place relative to the image when
+ using any "object-fit" values that are not "fill". To address this, use the `replacedContentRect()` of the image
+ renderer when injecting OCR text quads instead.
+
+ Tests: fast/images/text-recognition/image-overlay-object-fit-change.html
+ fast/images/text-recognition/image-overlay-object-fit.html
+
+ * html/HTMLElement.cpp:
+ (WebCore::HTMLElement::updateWithTextRecognitionResult): See description above for more details.
+ * page/Page.cpp:
+ (WebCore::Page::updateElementsWithTextRecognitionResults):
+
+ We also refactor this code so that we don't immediately try to update text recognition results, and instead
+ queue an internal async task to do so. Immediately performing the update here can potentially cause a debug
+ assertion due to leaving us in a state where we require layout immediately after a rendering update.
+
+ (WebCore::Page::cacheTextRecognitionResult):
+
+ Additionally adjust the text recognition result caching mechanism to update image overlay content when changing
+ the replaced content rect, rather than the element's offset width or height. This ensures that changing CSS
+ "object-fit" values dynamically for images with Live Text causes Live Text bounds to stay up to date.
+
+ * page/Page.h:
+
2021-06-28 Sam Weinig <[email protected]>
[Modern Media Controls] Support customizing the media controls via WebKitAdditions
Modified: trunk/Source/WebCore/html/HTMLElement.cpp (279348 => 279349)
--- trunk/Source/WebCore/html/HTMLElement.cpp 2021-06-28 21:24:48 UTC (rev 279348)
+++ trunk/Source/WebCore/html/HTMLElement.cpp 2021-06-28 21:44:51 UTC (rev 279349)
@@ -1456,13 +1456,6 @@
setInlineStyleProperty(CSSPropertyWebkitUserSelect, CSSValueText);
}
- if (auto* renderer = this->renderer()) {
- if (!is<RenderImage>(renderer))
- return;
-
- downcast<RenderImage>(*renderer).setHasImageOverlay();
- }
-
if (!hadExistingTextRecognitionElements) {
static MainThreadNeverDestroyed<const String> shadowStyle(StringImpl::createWithoutCopying(imageOverlayUserAgentStyleSheet, sizeof(imageOverlayUserAgentStyleSheet)));
auto style = HTMLStyleElement::create(HTMLNames::styleTag, document(), false);
@@ -1470,17 +1463,30 @@
shadowRoot->appendChild(WTFMove(style));
}
- IntSize containerSize { offsetWidth(), offsetHeight() };
+ document().updateLayoutIgnorePendingStylesheets();
+
+ auto* renderer = this->renderer();
+ if (!is<RenderImage>(renderer))
+ return;
+
+ downcast<RenderImage>(*renderer).setHasImageOverlay();
+
+ auto containerRect = enclosingIntRect(downcast<RenderImage>(*renderer).replacedContentRect());
+ auto convertToContainerCoordinates = [&](const FloatQuad& normalizedQuad) {
+ auto quad = normalizedQuad;
+ quad.scale(containerRect.width(), containerRect.height());
+ quad.move(containerRect.x(), containerRect.y());
+ return quad;
+ };
+
for (size_t lineIndex = 0; lineIndex < result.lines.size(); ++lineIndex) {
auto& lineElements = textRecognitionElements.lines[lineIndex];
auto& lineContainer = lineElements.line;
auto& line = result.lines[lineIndex];
- auto lineQuad = line.normalizedQuad;
+ auto lineQuad = convertToContainerCoordinates(line.normalizedQuad);
if (lineQuad.isEmpty())
continue;
- lineQuad.scale(containerSize.width(), containerSize.height());
-
auto lineBounds = rotatedBoundingRectWithMinimumAngleOfRotation(lineQuad, 0.01);
lineContainer->setInlineStyleProperty(CSSPropertyWidth, lineBounds.size.width(), CSSUnitType::CSS_PX);
lineContainer->setInlineStyleProperty(CSSPropertyHeight, lineBounds.size.height(), CSSUnitType::CSS_PX);
@@ -1500,8 +1506,7 @@
};
auto offsetsAlongHorizontalAxis = line.children.map([&](auto& child) -> WTF::Range<float> {
- auto textQuad = child.normalizedQuad;
- textQuad.scale(containerSize.width(), containerSize.height());
+ auto textQuad = convertToContainerCoordinates(child.normalizedQuad);
return {
offsetAlongHorizontalAxis(textQuad.p1(), textQuad.p4()),
offsetAlongHorizontalAxis(textQuad.p2(), textQuad.p3())
@@ -1575,9 +1580,7 @@
continue;
// FIXME: We should come up with a way to coalesce the bounding quads into one or more rotated rects with the same angle of rotation.
- auto targetQuad = dataDetector.normalizedQuads.first();
- targetQuad.scale(containerSize.width(), containerSize.height());
-
+ auto targetQuad = convertToContainerCoordinates(dataDetector.normalizedQuads.first());
auto targetBounds = rotatedBoundingRectWithMinimumAngleOfRotation(targetQuad, 0.01);
dataDetectorContainer->setInlineStyleProperty(CSSPropertyWidth, targetBounds.size.width(), CSSUnitType::CSS_PX);
dataDetectorContainer->setInlineStyleProperty(CSSPropertyHeight, targetBounds.size.height(), CSSUnitType::CSS_PX);
@@ -1595,7 +1598,7 @@
if (cacheTextRecognitionResults == CacheTextRecognitionResults::Yes) {
if (auto* page = document().page())
- page->cacheTextRecognitionResult(*this, containerSize, result);
+ page->cacheTextRecognitionResult(*this, containerRect, result);
}
}
Modified: trunk/Source/WebCore/page/Page.cpp (279348 => 279349)
--- trunk/Source/WebCore/page/Page.cpp 2021-06-28 21:24:48 UTC (rev 279348)
+++ trunk/Source/WebCore/page/Page.cpp 2021-06-28 21:44:51 UTC (rev 279349)
@@ -108,6 +108,7 @@
#include "ProgressTracker.h"
#include "Range.h"
#include "RenderDescendantIterator.h"
+#include "RenderImage.h"
#include "RenderLayerCompositor.h"
#include "RenderTheme.h"
#include "RenderView.h"
@@ -3618,19 +3619,34 @@
return !elementAndResult.first;
});
- for (auto& [element, resultAndSize] : m_textRecognitionResultsByElement) {
- if (!element || !element->isConnected())
+ Vector<std::pair<Ref<HTMLElement>, TextRecognitionResult>> elementsToUpdate;
+ for (auto& [element, resultAndRect] : m_textRecognitionResultsByElement) {
+ if (!element->isConnected())
continue;
+ auto& [result, containerRect] = resultAndRect;
auto protectedElement = makeRef(*element);
- auto& [result, offsetSize] = resultAndSize;
- IntSize newOffsetSize { protectedElement->offsetWidth(), protectedElement->offsetHeight() };
- if (offsetSize == newOffsetSize)
+ auto renderer = protectedElement->renderer();
+ if (!is<RenderImage>(renderer))
continue;
- offsetSize = newOffsetSize;
- protectedElement->updateWithTextRecognitionResult(result, HTMLElement::CacheTextRecognitionResults::No);
+ auto newContainerRect = enclosingIntRect(downcast<RenderImage>(*renderer).replacedContentRect());
+ if (containerRect == newContainerRect)
+ continue;
+
+ containerRect = newContainerRect;
+ elementsToUpdate.append({ WTFMove(protectedElement), result });
}
+
+ for (auto& [element, result] : elementsToUpdate) {
+ element->document().eventLoop().queueTask(TaskSource::InternalAsyncTask, [result = TextRecognitionResult { result }, weakElement = makeWeakPtr(element.get())] {
+ auto element = makeRefPtr(weakElement.get());
+ if (!element)
+ return;
+
+ element->updateWithTextRecognitionResult(result, HTMLElement::CacheTextRecognitionResults::No);
+ });
+ }
}
bool Page::hasCachedTextRecognitionResult(const HTMLElement& element) const
@@ -3638,7 +3654,7 @@
return m_elementsWithTextRecognitionResults.contains(element);
}
-void Page::cacheTextRecognitionResult(const HTMLElement& element, const IntSize& offsetSize, const TextRecognitionResult& result)
+void Page::cacheTextRecognitionResult(const HTMLElement& element, const IntRect& containerRect, const TextRecognitionResult& result)
{
m_elementsWithTextRecognitionResults.add(element);
@@ -3647,9 +3663,9 @@
});
if (index == notFound)
- m_textRecognitionResultsByElement.append({ makeWeakPtr(element), { result, offsetSize } });
+ m_textRecognitionResultsByElement.append({ makeWeakPtr(element), { result, containerRect } });
else
- m_textRecognitionResultsByElement[index].second = { result, offsetSize };
+ m_textRecognitionResultsByElement[index].second = { result, containerRect };
m_textRecognitionResultsByElement.removeAllMatching([](auto& elementAndResult) {
return !elementAndResult.first;
Modified: trunk/Source/WebCore/page/Page.h (279348 => 279349)
--- trunk/Source/WebCore/page/Page.h 2021-06-28 21:24:48 UTC (rev 279348)
+++ trunk/Source/WebCore/page/Page.h 2021-06-28 21:44:51 UTC (rev 279349)
@@ -863,7 +863,7 @@
#if ENABLE(IMAGE_ANALYSIS)
WEBCORE_EXPORT bool hasCachedTextRecognitionResult(const HTMLElement&) const;
- void cacheTextRecognitionResult(const HTMLElement&, const IntSize& offsetSize, const TextRecognitionResult&);
+ void cacheTextRecognitionResult(const HTMLElement&, const IntRect& containerRect, const TextRecognitionResult&);
#endif
private:
@@ -1189,7 +1189,7 @@
#if ENABLE(IMAGE_ANALYSIS)
// FIXME: These should be refactored to use a weak hash map of HTMLElement to std::pair<TextRecognitionResult, IntSize>.
- Vector<std::pair<WeakPtr<HTMLElement>, std::pair<TextRecognitionResult, IntSize>>> m_textRecognitionResultsByElement;
+ Vector<std::pair<WeakPtr<HTMLElement>, std::pair<TextRecognitionResult, IntRect>>> m_textRecognitionResultsByElement;
WeakHashSet<HTMLElement> m_elementsWithTextRecognitionResults;
#endif
};
