- Revision
- 292184
- Author
- wenson_hs...@apple.com
- Date
- 2022-03-31 14:58:55 -0700 (Thu, 31 Mar 2022)
Log Message
[iOS] Add support for temporarily installing VKCImageAnalysisInteraction on WKContentView
https://bugs.webkit.org/show_bug.cgi?id=238582
Reviewed by Tim Horton.
Source/WebCore/PAL:
Soft link against the image analysis interaction class.
* pal/cocoa/VisionKitCoreSoftLink.h:
* pal/cocoa/VisionKitCoreSoftLink.mm:
Source/WebKit:
Add helper methods for temporarily creating and installing an image analysis interaction on the root view. See
below for more details.
* UIProcess/ios/WKContentViewInteraction.h:
* UIProcess/ios/WKContentViewInteraction.mm:
(-[WKContentView hasSelectablePositionAtPoint:]):
(-[WKContentView pointIsNearMarkedText:]):
(-[WKContentView textInteractionGesture:shouldBeginAtPoint:]):
If Live Text exists at the given location, suppress default WebKit text selection behaviors in favor of Live
Text instead.
(-[WKContentView _singleTapRecognized:]):
Since we only allow the image analysis interaction to begin when interacting directly with regions of the
content view that contain Live Text, we need to (manually) clear the native text selection here when handling a
tap over other parts of the web view that are not covered by Live Text.
(-[WKContentView _didCommitLoadForMainFrame]):
Uninstall the interaction when loading a new page.
(-[WKContentView gestureRecognizer:shouldIgnoreWebTouchWithEvent:]):
(-[WKContentView deferringGestureRecognizer:willBeginTouchesWithEvent:]):
Don't allow touch events to even begin, if the gesture starts over a region of Live Text.
(-[WKContentView _tearDownImageAnalysis]):
Uninstall the interaction when the web process terminates or swaps.
(-[WKContentView installImageAnalysisInteraction:]):
(-[WKContentView uninstallImageAnalysisInteraction]):
These are the two main helper methods for adding or removing the image analysis interaction.
(-[WKContentView contentsRectForImageAnalysisInteraction:]):
(-[WKContentView imageAnalysisInteraction:shouldBeginAtPoint:forAnalysisType:]):
Only allow Live Text interaction gestures to begin if there's interactable content at the given location. The
only reason why this isn't default behavior already is that the image analysis interaction handles taps that
happen outside of Live Text by clearing any active selection; to ensure that this continues to work, we handle
the tap ourselves, and clear the native selection directly (see `-_singleTapRecognized:` above).
Modified Paths
Diff
Modified: trunk/Source/WebCore/PAL/ChangeLog (292183 => 292184)
--- trunk/Source/WebCore/PAL/ChangeLog 2022-03-31 21:56:59 UTC (rev 292183)
+++ trunk/Source/WebCore/PAL/ChangeLog 2022-03-31 21:58:55 UTC (rev 292184)
@@ -1,3 +1,15 @@
+2022-03-31 Wenson Hsieh <wenson_hs...@apple.com>
+
+ [iOS] Add support for temporarily installing VKCImageAnalysisInteraction on WKContentView
+ https://bugs.webkit.org/show_bug.cgi?id=238582
+
+ Reviewed by Tim Horton.
+
+ Soft link against the image analysis interaction class.
+
+ * pal/cocoa/VisionKitCoreSoftLink.h:
+ * pal/cocoa/VisionKitCoreSoftLink.mm:
+
2022-03-28 Simon Fraser <simon.fra...@apple.com>
[iOS GPUProcess] Image decoding should not trigger IOSurface usage in the web process
Modified: trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.h (292183 => 292184)
--- trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.h 2022-03-31 21:56:59 UTC (rev 292183)
+++ trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.h 2022-03-31 21:58:55 UTC (rev 292184)
@@ -36,6 +36,9 @@
SOFT_LINK_CLASS_FOR_HEADER(PAL, VKCImageAnalyzer)
SOFT_LINK_CLASS_FOR_HEADER(PAL, VKCImageAnalyzerRequest)
SOFT_LINK_CLASS_FOR_HEADER(PAL, VKCImageAnalysis)
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+SOFT_LINK_CLASS_FOR_HEADER(PAL, VKCImageAnalysisInteraction)
+#endif
#if USE(APPLE_INTERNAL_SDK)
#import <WebKitAdditions/VisionKitCoreSoftLinkAdditions.h>
Modified: trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.mm (292183 => 292184)
--- trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.mm 2022-03-31 21:56:59 UTC (rev 292183)
+++ trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.mm 2022-03-31 21:58:55 UTC (rev 292184)
@@ -36,6 +36,9 @@
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT_AND_IS_OPTIONAL(PAL, VisionKitCore, VKCImageAnalyzer, PAL_EXPORT, true)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT_AND_IS_OPTIONAL(PAL, VisionKitCore, VKCImageAnalyzerRequest, PAL_EXPORT, true)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT_AND_IS_OPTIONAL(PAL, VisionKitCore, VKCImageAnalysis, PAL_EXPORT, true)
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT_AND_IS_OPTIONAL(PAL, VisionKitCore, VKCImageAnalysisInteraction, PAL_EXPORT, true)
+#endif
#if USE(APPLE_INTERNAL_SDK)
#import <WebKitAdditions/VisionKitCoreSoftLinkAdditions.mm>
Modified: trunk/Source/WebKit/ChangeLog (292183 => 292184)
--- trunk/Source/WebKit/ChangeLog 2022-03-31 21:56:59 UTC (rev 292183)
+++ trunk/Source/WebKit/ChangeLog 2022-03-31 21:58:55 UTC (rev 292184)
@@ -1,3 +1,54 @@
+2022-03-31 Wenson Hsieh <wenson_hs...@apple.com>
+
+ [iOS] Add support for temporarily installing VKCImageAnalysisInteraction on WKContentView
+ https://bugs.webkit.org/show_bug.cgi?id=238582
+
+ Reviewed by Tim Horton.
+
+ Add helper methods for temporarily creating and installing an image analysis interaction on the root view. See
+ below for more details.
+
+ * UIProcess/ios/WKContentViewInteraction.h:
+ * UIProcess/ios/WKContentViewInteraction.mm:
+ (-[WKContentView hasSelectablePositionAtPoint:]):
+ (-[WKContentView pointIsNearMarkedText:]):
+ (-[WKContentView textInteractionGesture:shouldBeginAtPoint:]):
+
+ If Live Text exists at the given location, suppress default WebKit text selection behaviors in favor of Live
+ Text instead.
+
+ (-[WKContentView _singleTapRecognized:]):
+
+ Since we only allow the image analysis interaction to begin when interacting directly with regions of the
+ content view that contain Live Text, we need to (manually) clear the native text selection here when handling a
+ tap over other parts of the web view that are not covered by Live Text.
+
+ (-[WKContentView _didCommitLoadForMainFrame]):
+
+ Uninstall the interaction when loading a new page.
+
+ (-[WKContentView gestureRecognizer:shouldIgnoreWebTouchWithEvent:]):
+ (-[WKContentView deferringGestureRecognizer:willBeginTouchesWithEvent:]):
+
+ Don't allow touch events to even begin, if the gesture starts over a region of Live Text.
+
+ (-[WKContentView _tearDownImageAnalysis]):
+
+ Uninstall the interaction when the web process terminates or swaps.
+
+ (-[WKContentView installImageAnalysisInteraction:]):
+ (-[WKContentView uninstallImageAnalysisInteraction]):
+
+ These are the two main helper methods for adding or removing the image analysis interaction.
+
+ (-[WKContentView contentsRectForImageAnalysisInteraction:]):
+ (-[WKContentView imageAnalysisInteraction:shouldBeginAtPoint:forAnalysisType:]):
+
+ Only allow Live Text interaction gestures to begin if there's interactable content at the given location. The
+ only reason why this isn't default behavior already is that the image analysis interaction handles taps that
+ happen outside of Live Text by clearing any active selection; to ensure that this continues to work, we handle
+ the tap ourselves, and clear the native selection directly (see `-_singleTapRecognized:` above).
+
2022-03-31 Elliott Williams <e...@apple.com>
[XCBuild] WebKitLegacy's "Migrated headers" script does not emit task information
Modified: trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.h (292183 => 292184)
--- trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.h 2022-03-31 21:56:59 UTC (rev 292183)
+++ trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.h 2022-03-31 21:58:55 UTC (rev 292184)
@@ -112,6 +112,7 @@
@class AVPlayerViewController;
@class QLPreviewController;
+@class VKCImageAnalysisInteraction;
@class WebEvent;
@class WebTextIndicatorLayer;
@class WKActionSheetAssistant;
@@ -540,6 +541,8 @@
#endif // ENABLE(IMAGE_ANALYSIS)
uint32_t _fullscreenVideoExtractionRequestIdentifier;
#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ RetainPtr<VKCImageAnalysisInteraction> _imageAnalysisInteraction;
+ WebCore::FloatRect _imageAnalysisInteractionBounds;
std::optional<WebKit::ImageAnalysisMarkupData> _imageAnalysisMarkupData;
#endif
}
Modified: trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm (292183 => 292184)
--- trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm 2022-03-31 21:56:59 UTC (rev 292183)
+++ trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm 2022-03-31 21:58:55 UTC (rev 292184)
@@ -231,6 +231,13 @@
#endif // ENABLE(IMAGE_ANALYSIS)
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+
+@interface WKContentView (ImageAnalysisInteraction) <VKCImageAnalysisInteractionDelegate>
+@end
+
+#endif
+
#if USE(APPLE_INTERNAL_SDK)
#import <WebKitAdditions/WKContentViewInteractionAdditions.mm>
#else
@@ -2980,6 +2987,11 @@
- (BOOL)hasSelectablePositionAtPoint:(CGPoint)point
{
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ if ([_imageAnalysisInteraction interactableItemExistsAtPoint:point])
+ return NO;
+#endif
+
ALLOW_DEPRECATED_DECLARATIONS_BEGIN
if (!self.webView.configuration._textInteractionGesturesEnabled)
return NO;
@@ -3008,6 +3020,11 @@
- (BOOL)pointIsNearMarkedText:(CGPoint)point
{
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ if ([_imageAnalysisInteraction interactableItemExistsAtPoint:point])
+ return NO;
+#endif
+
ALLOW_DEPRECATED_DECLARATIONS_BEGIN
if (!self.webView.configuration._textInteractionGesturesEnabled)
return NO;
@@ -3027,6 +3044,11 @@
- (BOOL)textInteractionGesture:(UIWKGestureType)gesture shouldBeginAtPoint:(CGPoint)point
{
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ if ([_imageAnalysisInteraction interactableItemExistsAtPoint:point])
+ return NO;
+#endif
+
ALLOW_DEPRECATED_DECLARATIONS_BEGIN
if (!self.webView.configuration._textInteractionGesturesEnabled)
return NO;
@@ -3366,6 +3388,11 @@
if (!_isExpectingFastSingleTapCommit)
[self _finishInteraction];
+
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ if (![_imageAnalysisInteraction interactableItemExistsAtPoint:_lastInteractionLocation])
+ [_imageAnalysisInteraction resetSelection];
+#endif
}
- (void)_doubleTapRecognized:(UITapGestureRecognizer *)gestureRecognizer
@@ -5056,6 +5083,10 @@
#if HAVE(UIKIT_WITH_MOUSE_SUPPORT)
[_hoverPlatter dismissPlatterWithAnimation:NO];
#endif
+
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ [self uninstallImageAnalysisInteraction];
+#endif
}
- (void)_nextAccessoryTabForWebView:(id)sender
@@ -7877,6 +7908,11 @@
{
_touchEventsCanPreventNativeGestures = YES;
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ if ([_imageAnalysisInteraction interactableItemExistsAtPoint:[gestureRecognizer locationInView:self]])
+ return YES;
+#endif
+
return [self gestureRecognizer:gestureRecognizer isInterruptingMomentumScrollingWithEvent:event];
}
@@ -8287,6 +8323,11 @@
{
self.gestureRecognizerConsistencyEnforcer.beginTracking(deferringGestureRecognizer);
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ if ([_imageAnalysisInteraction interactableItemExistsAtPoint:[deferringGestureRecognizer locationInView:self]])
+ return WebKit::ShouldDeferGestures::No;
+#endif
+
return [self gestureRecognizer:deferringGestureRecognizer isInterruptingMomentumScrollingWithEvent:event] ? WebKit::ShouldDeferGestures::No : WebKit::ShouldDeferGestures::Yes;
}
@@ -10628,6 +10669,7 @@
#endif // USE(UICONTEXTMENU) && ENABLE(IMAGE_ANALYSIS_FOR_MACHINE_READABLE_CODES)
[self _invokeAllActionsToPerformAfterPendingImageAnalysis:WebKit::ProceedWithTextSelectionInImage::No];
#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ [self uninstallImageAnalysisInteraction];
_imageAnalysisMarkupData = std::nullopt;
#endif
}
@@ -10946,6 +10988,47 @@
});
}
+- (void)installImageAnalysisInteraction:(VKCImageAnalysis *)analysis
+{
+ if (!_imageAnalysisInteraction) {
+ _imageAnalysisInteraction = adoptNS([PAL::allocVKCImageAnalysisInteractionInstance() init]);
+ [_imageAnalysisInteraction setActiveInteractionTypes:VKImageAnalysisInteractionTypeTextSelection | VKImageAnalysisInteractionTypeDataDetectors];
+ [_imageAnalysisInteraction setDelegate:self];
+ [_imageAnalysisInteraction setWantsAutomaticContentsRectCalculation:NO];
+ [self addInteraction:_imageAnalysisInteraction.get()];
+ }
+ [_imageAnalysisInteraction setAnalysis:analysis];
+ [_imageAnalysisDeferringGestureRecognizer setEnabled:NO];
+ [_imageAnalysisGestureRecognizer setEnabled:NO];
+}
+
+- (void)uninstallImageAnalysisInteraction
+{
+ if (!_imageAnalysisInteraction)
+ return;
+
+ [self removeInteraction:_imageAnalysisInteraction.get()];
+ _imageAnalysisInteraction = nil;
+ [_imageAnalysisDeferringGestureRecognizer setEnabled:WebKit::isLiveTextAvailableAndEnabled()];
+ [_imageAnalysisGestureRecognizer setEnabled:WebKit::isLiveTextAvailableAndEnabled()];
+}
+
+#pragma mark - VKCImageAnalysisInteractionDelegate
+
+- (CGRect)contentsRectForImageAnalysisInteraction:(VKCImageAnalysisInteraction *)interaction
+{
+ auto unitInteractionRect = _imageAnalysisInteractionBounds;
+ WebCore::FloatRect unobscuredRect = self.bounds;
+ unitInteractionRect.moveBy(-unobscuredRect.location());
+ unitInteractionRect.scale(1 / unobscuredRect.size());
+ return unitInteractionRect;
+}
+
+- (BOOL)imageAnalysisInteraction:(VKCImageAnalysisInteraction *)interaction shouldBeginAtPoint:(CGPoint)point forAnalysisType:(VKImageAnalysisInteractionTypes)analysisType
+{
+ return [_imageAnalysisInteraction interactableItemExistsAtPoint:point];
+}
+
#endif // ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
#if USE(APPLE_INTERNAL_SDK)
