- Revision
- 285500
- Author
- [email protected]
- Date
- 2021-11-09 08:11:11 -0800 (Tue, 09 Nov 2021)
Log Message
[iOS] Add a position information bit to indicate whether the hit-tested element is a paused video
https://bugs.webkit.org/show_bug.cgi?id=232861
Reviewed by Megan Gardner.
Add `InteractionInformationAtPosition::isPausedVideo`, a flag that is true when the position information request
is over a paused video element (or inside the media control shadow root underneath that paused video element, in
the case where native controls are shown).
* Shared/ios/InteractionInformationAtPosition.h:
* Shared/ios/InteractionInformationAtPosition.mm:
(WebKit::InteractionInformationAtPosition::encode const):
(WebKit::InteractionInformationAtPosition::decode):
Also rename `imageElementContext` to `hostImageOrVideoElementContext` to clarify that it (1) may now include
element contexts for video elements, and (2) unlike the regular `elementContext`, this includes the image or
video element that is the host for hit-tested content in the UA shadow root of the element corresponding to
`elementContext`.
* UIProcess/ios/WKContentViewInteraction.mm:
(-[WKContentView hasSelectablePositionAtPoint:]):
(-[WKContentView textInteractionGesture:shouldBeginAtPoint:]):
(-[WKContentView imageAnalysisGestureDidBegin:]):
* WebProcess/WebPage/ios/WebPageIOS.mm:
(WebKit::videoPositionInformation):
Additionally populate the `image` of the position information, in the case where `includeImageData` is set on
the incoming request.
(WebKit::hostVideoElementIgnoringImageOverlay):
(WebKit::imagePositionInformation):
(WebKit::elementPositionInformation):
(WebKit::WebPage::positionInformation):
Modified Paths
Diff
Modified: trunk/Source/WebKit/ChangeLog (285499 => 285500)
--- trunk/Source/WebKit/ChangeLog 2021-11-09 16:08:12 UTC (rev 285499)
+++ trunk/Source/WebKit/ChangeLog 2021-11-09 16:11:11 UTC (rev 285500)
@@ -1,3 +1,39 @@
+2021-11-09 Wenson Hsieh <[email protected]>
+
+ [iOS] Add a position information bit to indicate whether the hit-tested element is a paused video
+ https://bugs.webkit.org/show_bug.cgi?id=232861
+
+ Reviewed by Megan Gardner.
+
+ Add `InteractionInformationAtPosition::isPausedVideo`, a flag that is true when the position information request
+ is over a paused video element (or inside the media control shadow root underneath that paused video element, in
+ the case where native controls are shown).
+
+ * Shared/ios/InteractionInformationAtPosition.h:
+ * Shared/ios/InteractionInformationAtPosition.mm:
+ (WebKit::InteractionInformationAtPosition::encode const):
+ (WebKit::InteractionInformationAtPosition::decode):
+
+ Also rename `imageElementContext` to `hostImageOrVideoElementContext` to clarify that it (1) may now include
+ element contexts for video elements, and (2) unlike the regular `elementContext`, this includes the image or
+ video element that is the host for hit-tested content in the UA shadow root of the element corresponding to
+ `elementContext`.
+
+ * UIProcess/ios/WKContentViewInteraction.mm:
+ (-[WKContentView hasSelectablePositionAtPoint:]):
+ (-[WKContentView textInteractionGesture:shouldBeginAtPoint:]):
+ (-[WKContentView imageAnalysisGestureDidBegin:]):
+ * WebProcess/WebPage/ios/WebPageIOS.mm:
+ (WebKit::videoPositionInformation):
+
+ Additionally populate the `image` of the position information, in the case where `includeImageData` is set on
+ the incoming request.
+
+ (WebKit::hostVideoElementIgnoringImageOverlay):
+ (WebKit::imagePositionInformation):
+ (WebKit::elementPositionInformation):
+ (WebKit::WebPage::positionInformation):
+
2021-11-08 Commit Queue <[email protected]>
Unreviewed, reverting r285432.
Modified: trunk/Source/WebKit/Shared/ios/InteractionInformationAtPosition.h (285499 => 285500)
--- trunk/Source/WebKit/Shared/ios/InteractionInformationAtPosition.h 2021-11-09 16:08:12 UTC (rev 285499)
+++ trunk/Source/WebKit/Shared/ios/InteractionInformationAtPosition.h 2021-11-09 16:11:11 UTC (rev 285500)
@@ -70,6 +70,7 @@
bool isImage { false };
bool isAttachment { false };
bool isAnimatedImage { false };
+ bool isPausedVideo { false };
bool isElement { false };
bool isContentEditable { false };
WebCore::ScrollingNodeID containerScrollingNodeID { 0 };
@@ -108,7 +109,7 @@
#endif
std::optional<WebCore::ElementContext> elementContext;
- std::optional<WebCore::ElementContext> imageElementContext;
+ std::optional<WebCore::ElementContext> hostImageOrVideoElementContext;
// Copy compatible optional bits forward (for example, if we have a InteractionInformationAtPosition
// with snapshots in it, and perform another request for the same point without requesting the snapshots,
Modified: trunk/Source/WebKit/Shared/ios/InteractionInformationAtPosition.mm (285499 => 285500)
--- trunk/Source/WebKit/Shared/ios/InteractionInformationAtPosition.mm 2021-11-09 16:08:12 UTC (rev 285499)
+++ trunk/Source/WebKit/Shared/ios/InteractionInformationAtPosition.mm 2021-11-09 16:11:11 UTC (rev 285500)
@@ -49,6 +49,7 @@
encoder << isImage;
encoder << isAttachment;
encoder << isAnimatedImage;
+ encoder << isPausedVideo;
encoder << isElement;
encoder << isContentEditable;
encoder << containerScrollingNodeID;
@@ -85,7 +86,7 @@
encoder << isImageOverlayText;
encoder << isVerticalWritingMode;
encoder << elementContext;
- encoder << imageElementContext;
+ encoder << hostImageOrVideoElementContext;
}
bool InteractionInformationAtPosition::decode(IPC::Decoder& decoder, InteractionInformationAtPosition& result)
@@ -125,6 +126,9 @@
if (!decoder.decode(result.isAnimatedImage))
return false;
+
+ if (!decoder.decode(result.isPausedVideo))
+ return false;
if (!decoder.decode(result.isElement))
return false;
@@ -220,7 +224,7 @@
if (!decoder.decode(result.elementContext))
return false;
- if (!decoder.decode(result.imageElementContext))
+ if (!decoder.decode(result.hostImageOrVideoElementContext))
return false;
return true;
Modified: trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm (285499 => 285500)
--- trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm 2021-11-09 16:08:12 UTC (rev 285499)
+++ trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm 2021-11-09 16:11:11 UTC (rev 285500)
@@ -2885,7 +2885,7 @@
return NO;
#if ENABLE(IMAGE_ANALYSIS)
- if (_elementPendingImageAnalysis && _positionInformation.imageElementContext == _elementPendingImageAnalysis)
+ if (_elementPendingImageAnalysis && _positionInformation.hostImageOrVideoElementContext == _elementPendingImageAnalysis)
return YES;
#endif
@@ -2966,7 +2966,7 @@
#endif
#if ENABLE(IMAGE_ANALYSIS)
- if (_elementPendingImageAnalysis && _positionInformation.imageElementContext == _elementPendingImageAnalysis)
+ if (_elementPendingImageAnalysis && _positionInformation.hostImageOrVideoElementContext == _elementPendingImageAnalysis)
return YES;
#endif
@@ -10193,7 +10193,7 @@
if (!information.image)
return false;
- if (!information.imageElementContext)
+ if (!information.hostImageOrVideoElementContext)
return false;
if (information.isAnimatedImage)
@@ -10218,10 +10218,10 @@
RELEASE_LOG(Images, "Image analysis preflight gesture initiated (request %" PRIu64 ").", requestIdentifier.toUInt64());
- strongSelf->_elementPendingImageAnalysis = information.imageElementContext;
+ strongSelf->_elementPendingImageAnalysis = information.hostImageOrVideoElementContext;
auto requestLocation = information.request.point;
- WebCore::ElementContext elementContext = *information.imageElementContext;
+ WebCore::ElementContext elementContext = *information.hostImageOrVideoElementContext;
auto requestForTextSelection = [strongSelf createImageAnalyzerRequest:VKAnalysisTypeText image:cgImage.get()];
auto requestForContextMenu = [strongSelf createImageAnalyzerRequest:VKAnalysisTypeVisualSearch | VKAnalysisTypeMachineReadableCode | VKAnalysisTypeAppClip image:cgImage.get()];
Modified: trunk/Source/WebKit/WebProcess/WebPage/ios/WebPageIOS.mm (285499 => 285500)
--- trunk/Source/WebKit/WebProcess/WebPage/ios/WebPageIOS.mm 2021-11-09 16:08:12 UTC (rev 285499)
+++ trunk/Source/WebKit/WebProcess/WebPage/ios/WebPageIOS.mm 2021-11-09 16:11:11 UTC (rev 285500)
@@ -105,6 +105,7 @@
#import <WebCore/HTMLSelectElement.h>
#import <WebCore/HTMLSummaryElement.h>
#import <WebCore/HTMLTextAreaElement.h>
+#import <WebCore/HTMLVideoElement.h>
#import <WebCore/HistoryItem.h>
#import <WebCore/HitTestResult.h>
#import <WebCore/InputMode.h>
@@ -129,6 +130,7 @@
#import <WebCore/RenderImage.h>
#import <WebCore/RenderLayer.h>
#import <WebCore/RenderThemeIOS.h>
+#import <WebCore/RenderVideo.h>
#import <WebCore/RenderView.h>
#import <WebCore/RenderedDocumentMarker.h>
#import <WebCore/RuntimeApplicationChecks.h>
@@ -2785,6 +2787,38 @@
return {{ renderImage, *image }};
}
+static void videoPositionInformation(WebPage& page, HTMLVideoElement& element, const InteractionInformationRequest& request, InteractionInformationAtPosition& info)
+{
+ if (!element.paused())
+ return;
+
+ auto renderVideo = element.renderer();
+ if (!renderVideo)
+ return;
+
+ info.isPausedVideo = true;
+
+ if (request.includeImageData)
+ info.image = createShareableBitmap(*renderVideo);
+
+ info.hostImageOrVideoElementContext = page.contextForElement(element);
+}
+
+static RefPtr<HTMLVideoElement> hostVideoElementIgnoringImageOverlay(Node& node)
+{
+ if (HTMLElement::isInsideImageOverlay(node))
+ return { };
+
+ if (is<HTMLVideoElement>(node))
+ return downcast<HTMLVideoElement>(&node);
+
+ RefPtr shadowHost = node.shadowHost();
+ if (!is<HTMLVideoElement>(shadowHost.get()))
+ return { };
+
+ return downcast<HTMLVideoElement>(shadowHost.get());
+}
+
static void imagePositionInformation(WebPage& page, Element& element, const InteractionInformationRequest& request, InteractionInformationAtPosition& info)
{
auto rendererAndImage = imageRendererAndImage(element);
@@ -2799,7 +2833,7 @@
if (request.includeSnapshot || request.includeImageData)
info.image = createShareableBitmap(renderImage, { screenSize() * page.corePage()->deviceScaleFactor(), AllowAnimatedImages::Yes, UseSnapshotForTransparentImages::Yes });
- info.imageElementContext = page.contextForElement(element);
+ info.hostImageOrVideoElementContext = page.contextForElement(element);
}
static void boundsPositionInformation(RenderObject& renderer, InteractionInformationAtPosition& info)
@@ -2863,8 +2897,12 @@
}
}
}
- if (shouldCollectImagePositionInformation)
- imagePositionInformation(page, element, request, info);
+ if (shouldCollectImagePositionInformation) {
+ if (auto video = hostVideoElementIgnoringImageOverlay(element))
+ videoPositionInformation(page, *video, request, info);
+ else
+ imagePositionInformation(page, element, request, info);
+ }
boundsPositionInformation(*renderer, info);
}
@@ -3100,8 +3138,12 @@
info.image = shareableBitmapSnapshotForNode(element);
}
- if (!info.isImage && request.includeImageData && is<HTMLImageElement>(hitTestNode))
- imagePositionInformation(*this, downcast<HTMLImageElement>(*hitTestNode), request, info);
+ if (!info.isImage && request.includeImageData) {
+ if (auto video = hostVideoElementIgnoringImageOverlay(*hitTestNode))
+ videoPositionInformation(*this, *video, request, info);
+ else if (is<HTMLImageElement>(hitTestNode))
+ imagePositionInformation(*this, downcast<HTMLImageElement>(*hitTestNode), request, info);
+ }
selectionPositionInformation(*this, request, info);
