- Revision
- 282740
- Author
- [email protected]
- Date
- 2021-09-19 17:10:37 -0700 (Sun, 19 Sep 2021)
Log Message
Teach `WebKit::createShareableBitmap` to snapshot video elements
https://bugs.webkit.org/show_bug.cgi?id=230468
Reviewed by Tim Horton.
Source/WebCore:
Now that `createShareableBitmap` may return images for video elements, we need to ensure that we explicitly
avoid allowing Live Text in video elements, since doing so will (1) break built-in platform media controls,
which also share the same shadow root, and (2) lead to confusing results when playing or seeking in videos,
since the recognized text falls out of sync with the video frame.
* html/HTMLVideoElement.h:
* page/EventHandler.cpp:
(WebCore::EventHandler::textRecognitionCandidateElement const):
For the above reasons, we refactor logic to check if the currently hovered node is a candidate for text
recognition, such that the video element check is consolidated all in one place.
(WebCore::EventHandler::updateMouseEventTargetNode):
(WebCore::EventHandler::textRecognitionHoverTimerFired):
* page/EventHandler.h:
* rendering/RenderVideo.h:
Source/WebKit:
`createShareableBitmap` currently only returns a non-null image bitmap for image renderers that have cached
images; notably, this omits video elements. For use in future patches, we should allow this helper function to
handle video elements by snapshotting the current frame of the video.
* WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp:
(WebKit::createShareableBitmap):
Modified Paths
Diff
Modified: trunk/Source/WebCore/ChangeLog (282739 => 282740)
--- trunk/Source/WebCore/ChangeLog 2021-09-19 23:46:05 UTC (rev 282739)
+++ trunk/Source/WebCore/ChangeLog 2021-09-20 00:10:37 UTC (rev 282740)
@@ -1,3 +1,27 @@
+2021-09-19 Wenson Hsieh <[email protected]>
+
+ Teach `WebKit::createShareableBitmap` to snapshot video elements
+ https://bugs.webkit.org/show_bug.cgi?id=230468
+
+ Reviewed by Tim Horton.
+
+ Now that `createShareableBitmap` may return images for video elements, we need to ensure that we explicitly
+ avoid allowing Live Text in video elements, since doing so will (1) break built-in platform media controls,
+ which also share the same shadow root, and (2) lead to confusing results when playing or seeking in videos,
+ since the recognized text falls out of sync with the video frame.
+
+ * html/HTMLVideoElement.h:
+ * page/EventHandler.cpp:
+ (WebCore::EventHandler::textRecognitionCandidateElement const):
+
+ For the above reasons, we refactor logic to check if the currently hovered node is a candidate for text
+ recognition, such that the video element check is consolidated all in one place.
+
+ (WebCore::EventHandler::updateMouseEventTargetNode):
+ (WebCore::EventHandler::textRecognitionHoverTimerFired):
+ * page/EventHandler.h:
+ * rendering/RenderVideo.h:
+
2021-09-19 Alan Bujtas <[email protected]>
[LFC][IFC] overflow-wrap: anywhere/break-word rules over word-break: keep-all
Modified: trunk/Source/WebCore/html/HTMLVideoElement.h (282739 => 282740)
--- trunk/Source/WebCore/html/HTMLVideoElement.h 2021-09-19 23:46:05 UTC (rev 282739)
+++ trunk/Source/WebCore/html/HTMLVideoElement.h 2021-09-20 00:10:37 UTC (rev 282740)
@@ -77,7 +77,7 @@
// Used by canvas to gain raw pixel access
void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&);
- RefPtr<NativeImage> nativeImageForCurrentTime();
+ WEBCORE_EXPORT RefPtr<NativeImage> nativeImageForCurrentTime();
WEBCORE_EXPORT bool shouldDisplayPosterImage() const;
Modified: trunk/Source/WebCore/page/EventHandler.cpp (282739 => 282740)
--- trunk/Source/WebCore/page/EventHandler.cpp 2021-09-19 23:46:05 UTC (rev 282739)
+++ trunk/Source/WebCore/page/EventHandler.cpp 2021-09-20 00:10:37 UTC (rev 282740)
@@ -63,6 +63,7 @@
#include "HTMLIFrameElement.h"
#include "HTMLInputElement.h"
#include "HTMLNames.h"
+#include "HTMLVideoElement.h"
#include "HitTestRequest.h"
#include "HitTestResult.h"
#include "Image.h"
@@ -83,6 +84,7 @@
#include "PluginDocument.h"
#include "Range.h"
#include "RenderFrameSet.h"
+#include "RenderImage.h"
#include "RenderLayer.h"
#include "RenderLayerScrollableArea.h"
#include "RenderListBox.h"
@@ -2512,6 +2514,22 @@
return false;
}
+RefPtr<Element> EventHandler::textRecognitionCandidateElement() const
+{
+ RefPtr shadowHost = m_elementUnderMouse ? m_elementUnderMouse->shadowHost() : nullptr;
+ if (!shadowHost)
+ return nullptr;
+
+ auto renderer = shadowHost->renderer();
+ if (!is<RenderImage>(renderer))
+ return nullptr;
+
+ if (is<HTMLVideoElement>(*shadowHost))
+ return nullptr;
+
+ return shadowHost;
+}
+
void EventHandler::updateMouseEventTargetNode(const AtomString& eventType, Node* targetNode, const PlatformMouseEvent& platformMouseEvent, FireMouseOverOut fireMouseOverOut)
{
Ref<Frame> protectedFrame(m_frame);
@@ -2531,7 +2549,7 @@
#if ENABLE(IMAGE_ANALYSIS)
if (m_frame.settings().preferInlineTextSelectionInImages()) {
- if (!m_elementUnderMouse || !is<RenderImage>(m_elementUnderMouse->renderer()))
+ if (!textRecognitionCandidateElement())
m_textRecognitionHoverTimer.stop();
else if (!platformMouseEvent.movementDelta().isZero())
m_textRecognitionHoverTimer.restart();
@@ -3410,11 +3428,12 @@
void EventHandler::textRecognitionHoverTimerFired()
{
- if (!m_elementUnderMouse || !is<RenderImage>(m_elementUnderMouse->renderer()))
+ auto element = this->textRecognitionCandidateElement();
+ if (!element)
return;
if (auto* page = m_frame.page())
- page->chrome().client().requestTextRecognition(*m_elementUnderMouse);
+ page->chrome().client().requestTextRecognition(*element);
}
#endif // ENABLE(IMAGE_ANALYSIS)
Modified: trunk/Source/WebCore/page/EventHandler.h (282739 => 282740)
--- trunk/Source/WebCore/page/EventHandler.h 2021-09-19 23:46:05 UTC (rev 282739)
+++ trunk/Source/WebCore/page/EventHandler.h 2021-09-20 00:10:37 UTC (rev 282740)
@@ -363,6 +363,8 @@
static const Seconds TextDragDelay;
#endif
+ RefPtr<Element> textRecognitionCandidateElement() const;
+
bool eventActivatedView(const PlatformMouseEvent&) const;
bool updateSelectionForMouseDownDispatchingSelectStart(Node*, const VisibleSelection&, TextGranularity);
void selectClosestWordFromHitTestResult(const HitTestResult&, AppendTrailingWhitespace);
Modified: trunk/Source/WebCore/rendering/RenderVideo.h (282739 => 282740)
--- trunk/Source/WebCore/rendering/RenderVideo.h 2021-09-19 23:46:05 UTC (rev 282739)
+++ trunk/Source/WebCore/rendering/RenderVideo.h 2021-09-20 00:10:37 UTC (rev 282740)
@@ -38,7 +38,7 @@
RenderVideo(HTMLVideoElement&, RenderStyle&&);
virtual ~RenderVideo();
- HTMLVideoElement& videoElement() const;
+ WEBCORE_EXPORT HTMLVideoElement& videoElement() const;
WEBCORE_EXPORT IntRect videoBox() const;
Modified: trunk/Source/WebKit/ChangeLog (282739 => 282740)
--- trunk/Source/WebKit/ChangeLog 2021-09-19 23:46:05 UTC (rev 282739)
+++ trunk/Source/WebKit/ChangeLog 2021-09-20 00:10:37 UTC (rev 282740)
@@ -1,3 +1,17 @@
+2021-09-19 Wenson Hsieh <[email protected]>
+
+ Teach `WebKit::createShareableBitmap` to snapshot video elements
+ https://bugs.webkit.org/show_bug.cgi?id=230468
+
+ Reviewed by Tim Horton.
+
+ `createShareableBitmap` currently only returns a non-null image bitmap for image renderers that have cached
+ images; notably, this omits video elements. For use in future patches, we should allow this helper function to
+ handle video elements by snapshotting the current frame of the video.
+
+ * WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp:
+ (WebKit::createShareableBitmap):
+
2021-09-17 Alex Christensen <[email protected]>
Use ObjectIdentifier for ResourceLoader
Modified: trunk/Source/WebKit/WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp (282739 => 282740)
--- trunk/Source/WebKit/WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp 2021-09-19 23:46:05 UTC (rev 282739)
+++ trunk/Source/WebKit/WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp 2021-09-20 00:10:37 UTC (rev 282740)
@@ -36,6 +36,7 @@
#include <WebCore/IntSize.h>
#include <WebCore/PlatformScreen.h>
#include <WebCore/RenderImage.h>
+#include <WebCore/RenderVideo.h>
namespace WebKit {
using namespace WebCore;
@@ -70,6 +71,29 @@
return bitmap;
}
+ if (is<RenderVideo>(renderImage)) {
+ auto& renderVideo = downcast<RenderVideo>(renderImage);
+ Ref video = renderVideo.videoElement();
+ auto image = video->nativeImageForCurrentTime();
+ if (!image)
+ return { };
+
+ auto imageSize = image->size();
+ if (imageSize.isEmpty() || imageSize.width() <= 1 || imageSize.height() <= 1)
+ return { };
+
+ auto bitmap = ShareableBitmap::createShareable(imageSize, { WTFMove(colorSpaceForBitmap) });
+ if (!bitmap)
+ return { };
+
+ auto context = bitmap->createGraphicsContext();
+ if (!context)
+ return { };
+
+ context->drawNativeImage(*image, imageSize, FloatRect { { }, imageSize }, FloatRect { { }, imageSize });
+ return bitmap;
+ }
+
auto* cachedImage = renderImage.cachedImage();
if (!cachedImage || cachedImage->errorOccurred())
return { };
