Diff
Modified: trunk/LayoutTests/ChangeLog (279750 => 279751)
--- trunk/LayoutTests/ChangeLog 2021-07-08 20:54:04 UTC (rev 279750)
+++ trunk/LayoutTests/ChangeLog 2021-07-08 20:59:32 UTC (rev 279751)
@@ -1,3 +1,14 @@
+2021-07-08 Wenson Hsieh <[email protected]>
+
+ [Live Text] Selection is misaligned on some images on twitter.com
+ https://bugs.webkit.org/show_bug.cgi?id=227775
+ rdar://77142364
+
+ Reviewed by Tim Horton.
+
+ * fast/images/text-recognition/image-overlay-in-transparent-image-expected.txt: Added.
+ * fast/images/text-recognition/image-overlay-in-transparent-image.html: Added.
+
2021-07-08 Kate Cheney <[email protected]>
Clean up App Privacy Report code
Added: trunk/LayoutTests/fast/images/text-recognition/image-overlay-in-transparent-image-expected.txt (0 => 279751)
--- trunk/LayoutTests/fast/images/text-recognition/image-overlay-in-transparent-image-expected.txt (rev 0)
+++ trunk/LayoutTests/fast/images/text-recognition/image-overlay-in-transparent-image-expected.txt 2021-07-08 20:59:32 UTC (rev 279751)
@@ -0,0 +1,6 @@
+PASS boundingRect.width is 250
+PASS boundingRect.height is 200
+PASS successfullyParsed is true
+
+TEST COMPLETE
+
Added: trunk/LayoutTests/fast/images/text-recognition/image-overlay-in-transparent-image.html (0 => 279751)
--- trunk/LayoutTests/fast/images/text-recognition/image-overlay-in-transparent-image.html (rev 0)
+++ trunk/LayoutTests/fast/images/text-recognition/image-overlay-in-transparent-image.html 2021-07-08 20:59:32 UTC (rev 279751)
@@ -0,0 +1,54 @@
+<!DOCTYPE html>
+<html>
+<head>
+<style>
+img {
+ position: absolute;
+ top: 0;
+ left: 0;
+ width: 250px;
+ height: 400px;
+}
+
+.back {
+ object-fit: cover;
+}
+
+.front {
+ opacity: 0;
+}
+</style>
+<script src=""
+</head>
+<body>
+<img class="back" src=""
+<img class="front" src=""
+<script>
+addEventListener("load", () => {
+ const image = document.querySelector("img.front");
+ internals.installImageOverlay(image, [
+ {
+ topLeft : new DOMPointReadOnly(0, 0),
+ topRight : new DOMPointReadOnly(1, 0),
+ bottomRight : new DOMPointReadOnly(1, 0.5),
+ bottomLeft : new DOMPointReadOnly(0, 0.5),
+ children: [
+ {
+ text : "foo",
+ topLeft : new DOMPointReadOnly(0, 0),
+ topRight : new DOMPointReadOnly(1, 0),
+ bottomRight : new DOMPointReadOnly(1, 0.5),
+ bottomLeft : new DOMPointReadOnly(0, 0.5),
+ }
+ ],
+ }
+ ]);
+
+ let text = internals.shadowRoot(image).querySelector(".image-overlay-text");
+ boundingRect = text.getBoundingClientRect();
+ shouldBe("boundingRect.width", "250");
+ shouldBe("boundingRect.height", "200");
+});
+</script>
+</body>
+</html>
\ No newline at end of file
Modified: trunk/Source/WebCore/ChangeLog (279750 => 279751)
--- trunk/Source/WebCore/ChangeLog 2021-07-08 20:54:04 UTC (rev 279750)
+++ trunk/Source/WebCore/ChangeLog 2021-07-08 20:59:32 UTC (rev 279751)
@@ -1,3 +1,30 @@
+2021-07-08 Wenson Hsieh <[email protected]>
+
+ [Live Text] Selection is misaligned on some images on twitter.com
+ https://bugs.webkit.org/show_bug.cgi?id=227775
+ rdar://77142364
+
+ Reviewed by Tim Horton.
+
+ On Twitter, image thumbnails consist of fully transparent image elements (`opacity: 0;`) that are overlaid over
+ `div` elements with a background image. The images for some of these transparent image elements sometimes don't
+ match the background image behind it (namely, the image is stretched to exactly fit the bounds of the
+ transparent image element in a way that does not preserve aspect ratio, whereas the background image is cropped
+ to cover the `div`). As such, Live Text in these image elements ends up misaligned against the background image
+ (which is actually visible to the user).
+
+ To address this, special case fully transparent images such that we take a snapshot of the page using the
+ absolute bounds of the transparent image and run OCR over this snapshot, instead of using the image data as-is.
+
+ Test: fast/images/text-recognition/image-overlay-in-transparent-image.html
+
+ * html/HTMLElement.cpp:
+ (WebCore::HTMLElement::containerRectForTextRecognition):
+ (WebCore::HTMLElement::updateWithTextRecognitionResult):
+ * html/HTMLElement.h:
+ * page/Page.cpp:
+ (WebCore::Page::updateElementsWithTextRecognitionResults):
+
2021-07-08 Kate Cheney <[email protected]>
Clean up App Privacy Report code
Modified: trunk/Source/WebCore/html/HTMLElement.cpp (279750 => 279751)
--- trunk/Source/WebCore/html/HTMLElement.cpp 2021-07-08 20:54:04 UTC (rev 279750)
+++ trunk/Source/WebCore/html/HTMLElement.cpp 2021-07-08 20:59:32 UTC (rev 279751)
@@ -1340,6 +1340,18 @@
#if ENABLE(IMAGE_ANALYSIS)
+IntRect HTMLElement::containerRectForTextRecognition()
+{
+ auto* renderer = this->renderer();
+ if (!is<RenderImage>(renderer))
+ return { };
+
+ if (!renderer->opacity())
+ return { 0, 0, offsetWidth(), offsetHeight() };
+
+ return enclosingIntRect(downcast<RenderImage>(*renderer).replacedContentRect());
+}
+
void HTMLElement::updateWithTextRecognitionResult(const TextRecognitionResult& result, CacheTextRecognitionResults cacheTextRecognitionResults)
{
static MainThreadNeverDestroyed<const AtomString> imageOverlayLineClass("image-overlay-line", AtomString::ConstructFromLiteral);
@@ -1472,7 +1484,7 @@
downcast<RenderImage>(*renderer).setHasImageOverlay();
- auto containerRect = enclosingIntRect(downcast<RenderImage>(*renderer).replacedContentRect());
+ auto containerRect = containerRectForTextRecognition();
auto convertToContainerCoordinates = [&](const FloatQuad& normalizedQuad) {
auto quad = normalizedQuad;
quad.scale(containerRect.width(), containerRect.height());
Modified: trunk/Source/WebCore/html/HTMLElement.h (279750 => 279751)
--- trunk/Source/WebCore/html/HTMLElement.h 2021-07-08 20:54:04 UTC (rev 279750)
+++ trunk/Source/WebCore/html/HTMLElement.h 2021-07-08 20:59:32 UTC (rev 279751)
@@ -139,6 +139,7 @@
WEBCORE_EXPORT static bool isImageOverlayText(const Node*);
#if ENABLE(IMAGE_ANALYSIS)
+ IntRect containerRectForTextRecognition();
enum class CacheTextRecognitionResults : bool { No, Yes };
WEBCORE_EXPORT void updateWithTextRecognitionResult(const TextRecognitionResult&, CacheTextRecognitionResults = CacheTextRecognitionResults::Yes);
#endif
Modified: trunk/Source/WebCore/page/Page.cpp (279750 => 279751)
--- trunk/Source/WebCore/page/Page.cpp 2021-07-08 20:54:04 UTC (rev 279750)
+++ trunk/Source/WebCore/page/Page.cpp 2021-07-08 20:59:32 UTC (rev 279751)
@@ -3630,7 +3630,7 @@
if (!is<RenderImage>(renderer))
continue;
- auto newContainerRect = enclosingIntRect(downcast<RenderImage>(*renderer).replacedContentRect());
+ auto newContainerRect = protectedElement->containerRectForTextRecognition();
if (containerRect == newContainerRect)
continue;
Modified: trunk/Source/WebKit/ChangeLog (279750 => 279751)
--- trunk/Source/WebKit/ChangeLog 2021-07-08 20:54:04 UTC (rev 279750)
+++ trunk/Source/WebKit/ChangeLog 2021-07-08 20:59:32 UTC (rev 279751)
@@ -1,3 +1,27 @@
+2021-07-08 Wenson Hsieh <[email protected]>
+
+ [Live Text] Selection is misaligned on some images on twitter.com
+ https://bugs.webkit.org/show_bug.cgi?id=227775
+ rdar://77142364
+
+ Reviewed by Tim Horton.
+
+ See WebCore ChangeLog for more details.
+
+ * WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp:
+ (WebKit::createShareableBitmap):
+ * WebProcess/WebCoreSupport/ShareableBitmapUtilities.h:
+ (WebKit::createShareableBitmap):
+
+ Add an option to handle the scenario where the image renderer is fully transparent by falling back to a frame-
+ level snapshot, using the bounds of the image renderer.
+
+ * WebProcess/WebPage/WebPage.cpp:
+ (WebKit::WebPage::requestTextRecognition):
+ * WebProcess/WebPage/ios/WebPageIOS.mm:
+ (WebKit::imagePositionInformation):
+ (WebKit::elementPositionInformation):
+
2021-07-08 Kate Cheney <[email protected]>
Clean up App Privacy Report code
Modified: trunk/Source/WebKit/WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp (279750 => 279751)
--- trunk/Source/WebKit/WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp 2021-07-08 20:54:04 UTC (rev 279750)
+++ trunk/Source/WebKit/WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp 2021-07-08 20:59:32 UTC (rev 279751)
@@ -28,8 +28,11 @@
#include "ShareableBitmap.h"
#include <WebCore/CachedImage.h>
+#include <WebCore/Frame.h>
+#include <WebCore/FrameSnapshotting.h>
#include <WebCore/GeometryUtilities.h>
#include <WebCore/GraphicsContext.h>
+#include <WebCore/ImageBuffer.h>
#include <WebCore/IntSize.h>
#include <WebCore/PlatformScreen.h>
#include <WebCore/RenderImage.h>
@@ -37,33 +40,61 @@
namespace WebKit {
using namespace WebCore;
-RefPtr<ShareableBitmap> createShareableBitmap(RenderImage& renderImage, std::optional<FloatSize> screenSizeInPixels, AllowAnimatedImages allowAnimatedImages)
+RefPtr<ShareableBitmap> createShareableBitmap(RenderImage& renderImage, CreateShareableBitmapFromImageOptions&& options)
{
+ Ref frame = renderImage.frame();
+ auto colorSpaceForBitmap = screenColorSpace(frame->mainFrame().view());
+ if (!renderImage.opacity() && options.useSnapshotForTransparentImages == UseSnapshotForTransparentImages::Yes) {
+ auto snapshotRect = renderImage.absoluteBoundingBoxRect();
+ if (snapshotRect.isEmpty())
+ return { };
+
+ OptionSet<SnapshotFlags> snapshotFlags { SnapshotFlags::ExcludeSelectionHighlighting, SnapshotFlags::PaintEverythingExcludingSelection };
+ auto imageBuffer = snapshotFrameRect(frame.get(), snapshotRect, { snapshotFlags, PixelFormat::BGRA8, DestinationColorSpace::SRGB() });
+ if (!imageBuffer)
+ return { };
+
+ auto snapshotImage = ImageBuffer::sinkIntoImage(WTFMove(imageBuffer), PreserveResolution::Yes);
+ if (!snapshotImage)
+ return { };
+
+ auto bitmap = ShareableBitmap::createShareable(snapshotRect.size(), { WTFMove(colorSpaceForBitmap) });
+ if (!bitmap)
+ return { };
+
+ auto context = bitmap->createGraphicsContext();
+ if (!context)
+ return { };
+
+ context->drawImage(*snapshotImage, { FloatPoint::zero(), snapshotRect.size() });
+ return bitmap;
+ }
+
auto* cachedImage = renderImage.cachedImage();
if (!cachedImage || cachedImage->errorOccurred())
- return nullptr;
+ return { };
auto* image = cachedImage->imageForRenderer(&renderImage);
if (!image || image->width() <= 1 || image->height() <= 1)
- return nullptr;
+ return { };
- if (allowAnimatedImages == AllowAnimatedImages::No && image->isAnimated())
- return nullptr;
+ if (options.allowAnimatedImages == AllowAnimatedImages::No && image->isAnimated())
+ return { };
auto bitmapSize = cachedImage->imageSizeForRenderer(&renderImage);
- if (screenSizeInPixels) {
- auto scaledSize = largestRectWithAspectRatioInsideRect(bitmapSize.width() / bitmapSize.height(), { FloatPoint(), *screenSizeInPixels }).size();
+ if (options.screenSizeInPixels) {
+ auto scaledSize = largestRectWithAspectRatioInsideRect(bitmapSize.width() / bitmapSize.height(), { FloatPoint(), *options.screenSizeInPixels }).size();
bitmapSize = scaledSize.width() < bitmapSize.width() ? scaledSize : bitmapSize;
}
// FIXME: Only select ExtendedColor on images known to need wide gamut.
- auto sharedBitmap = ShareableBitmap::createShareable(IntSize(bitmapSize), { screenColorSpace(renderImage.frame().mainFrame().view()) });
+ auto sharedBitmap = ShareableBitmap::createShareable(IntSize(bitmapSize), { WTFMove(colorSpaceForBitmap) });
if (!sharedBitmap)
- return nullptr;
+ return { };
auto graphicsContext = sharedBitmap->createGraphicsContext();
if (!graphicsContext)
- return nullptr;
+ return { };
graphicsContext->drawImage(*image, FloatRect(0, 0, bitmapSize.width(), bitmapSize.height()), { renderImage.imageOrientation() });
return sharedBitmap;
Modified: trunk/Source/WebKit/WebProcess/WebCoreSupport/ShareableBitmapUtilities.h (279750 => 279751)
--- trunk/Source/WebKit/WebProcess/WebCoreSupport/ShareableBitmapUtilities.h 2021-07-08 20:54:04 UTC (rev 279750)
+++ trunk/Source/WebKit/WebProcess/WebCoreSupport/ShareableBitmapUtilities.h 2021-07-08 20:59:32 UTC (rev 279751)
@@ -36,7 +36,15 @@
class ShareableBitmap;
+enum class UseSnapshotForTransparentImages : bool { No, Yes };
enum class AllowAnimatedImages : bool { No, Yes };
-RefPtr<ShareableBitmap> createShareableBitmap(WebCore::RenderImage&, std::optional<WebCore::FloatSize> screenSizeInPixels = std::nullopt, AllowAnimatedImages = AllowAnimatedImages::Yes);
+struct CreateShareableBitmapFromImageOptions {
+ std::optional<WebCore::FloatSize> screenSizeInPixels;
+ AllowAnimatedImages allowAnimatedImages { AllowAnimatedImages::Yes };
+ UseSnapshotForTransparentImages useSnapshotForTransparentImages { UseSnapshotForTransparentImages::No };
};
+
+RefPtr<ShareableBitmap> createShareableBitmap(WebCore::RenderImage&, CreateShareableBitmapFromImageOptions&& = { });
+
+};
Modified: trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp (279750 => 279751)
--- trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp 2021-07-08 20:54:04 UTC (rev 279750)
+++ trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp 2021-07-08 20:59:32 UTC (rev 279751)
@@ -7406,7 +7406,7 @@
}
auto& renderImage = downcast<RenderImage>(*renderer);
- auto bitmap = createShareableBitmap(renderImage, std::nullopt, AllowAnimatedImages::No);
+ auto bitmap = createShareableBitmap(renderImage, { std::nullopt, AllowAnimatedImages::No, UseSnapshotForTransparentImages::Yes });
if (!bitmap) {
if (completion)
completion({ });
Modified: trunk/Source/WebKit/WebProcess/WebPage/ios/WebPageIOS.mm (279750 => 279751)
--- trunk/Source/WebKit/WebProcess/WebPage/ios/WebPageIOS.mm 2021-07-08 20:54:04 UTC (rev 279750)
+++ trunk/Source/WebKit/WebProcess/WebPage/ios/WebPageIOS.mm 2021-07-08 20:59:32 UTC (rev 279751)
@@ -2823,7 +2823,7 @@
info.isAnimatedImage = image.isAnimated();
if (request.includeSnapshot || request.includeImageData)
- info.image = createShareableBitmap(renderImage, screenSize() * page.corePage()->deviceScaleFactor());
+ info.image = createShareableBitmap(renderImage, { screenSize() * page.corePage()->deviceScaleFactor(), AllowAnimatedImages::Yes, UseSnapshotForTransparentImages::Yes });
info.imageElementContext = page.contextForElement(element);
}
@@ -2885,7 +2885,7 @@
if (auto rendererAndImage = imageRendererAndImage(element)) {
auto& [renderImage, image] = *rendererAndImage;
info.imageURL = element.document().completeURL(renderImage.cachedImage()->url().string());
- info.image = createShareableBitmap(renderImage, screenSize() * page.corePage()->deviceScaleFactor());
+ info.image = createShareableBitmap(renderImage, { screenSize() * page.corePage()->deviceScaleFactor(), AllowAnimatedImages::Yes, UseSnapshotForTransparentImages::Yes });
}
}
}
