Diff
Modified: trunk/Source/WebCore/PAL/ChangeLog (292315 => 292316)
--- trunk/Source/WebCore/PAL/ChangeLog 2022-04-04 20:54:01 UTC (rev 292315)
+++ trunk/Source/WebCore/PAL/ChangeLog 2022-04-04 21:16:05 UTC (rev 292316)
@@ -1,3 +1,15 @@
+2022-04-04 Wenson Hsieh <[email protected]>
+
+ [macOS] Add helper methods to install and uninstall VKCImageAnalysisOverlayView
+ https://bugs.webkit.org/show_bug.cgi?id=238714
+
+ Reviewed by Tim Horton.
+
+ Add soft-linking support for VKCImageAnalysisOverlayView. See WebKit/ChangeLog for more details.
+
+ * pal/cocoa/VisionKitCoreSoftLink.h:
+ * pal/cocoa/VisionKitCoreSoftLink.mm:
+
2022-04-03 Tim Horton <[email protected]>
_WKDataTask doesn't work in macCatalyst
Modified: trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.h (292315 => 292316)
--- trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.h 2022-04-04 20:54:01 UTC (rev 292315)
+++ trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.h 2022-04-04 21:16:05 UTC (rev 292316)
@@ -38,6 +38,7 @@
SOFT_LINK_CLASS_FOR_HEADER(PAL, VKCImageAnalysis)
#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
SOFT_LINK_CLASS_FOR_HEADER(PAL, VKCImageAnalysisInteraction)
+SOFT_LINK_CLASS_FOR_HEADER(PAL, VKCImageAnalysisOverlayView)
#endif
#if USE(APPLE_INTERNAL_SDK)
Modified: trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.mm (292315 => 292316)
--- trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.mm 2022-04-04 20:54:01 UTC (rev 292315)
+++ trunk/Source/WebCore/PAL/pal/cocoa/VisionKitCoreSoftLink.mm 2022-04-04 21:16:05 UTC (rev 292316)
@@ -38,6 +38,7 @@
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT_AND_IS_OPTIONAL(PAL, VisionKitCore, VKCImageAnalysis, PAL_EXPORT, true)
#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT_AND_IS_OPTIONAL(PAL, VisionKitCore, VKCImageAnalysisInteraction, PAL_EXPORT, true)
+SOFT_LINK_CLASS_FOR_SOURCE_WITH_EXPORT_AND_IS_OPTIONAL(PAL, VisionKitCore, VKCImageAnalysisOverlayView, PAL_EXPORT, true)
#endif
#if USE(APPLE_INTERNAL_SDK)
Modified: trunk/Source/WebKit/ChangeLog (292315 => 292316)
--- trunk/Source/WebKit/ChangeLog 2022-04-04 20:54:01 UTC (rev 292315)
+++ trunk/Source/WebKit/ChangeLog 2022-04-04 21:16:05 UTC (rev 292316)
@@ -1,3 +1,51 @@
+2022-04-04 Wenson Hsieh <[email protected]>
+
+ [macOS] Add helper methods to install and uninstall VKCImageAnalysisOverlayView
+ https://bugs.webkit.org/show_bug.cgi?id=238714
+
+ Reviewed by Tim Horton.
+
+ Add helper methods for adding and removing a temporary VKCImageAnalysisOverlayView as a subview of WKWebView.
+ See below for more details. No change in behavior (yet).
+
+ * UIProcess/Cocoa/WebViewImpl.h:
+ (WebKit::WebViewImpl::imageAnalysisInteractionBounds const):
+ (WebKit::WebViewImpl::imageAnalysisOverlayView const):
+ * UIProcess/Cocoa/WebViewImpl.mm:
+ (WebKit::WebViewImpl::processImageAnalyzerRequest):
+
+ Pull logic for processing an image analyzer request and invoking the completion handler on the main thread (via
+ `callOnMainRunLoop`) out into a separate helper method (to be used in a subsequent patch).
+
+ (WebKit::WebViewImpl::requestTextRecognition):
+ (-[WKImageAnalysisOverlayViewDelegate initWithWebViewImpl:]):
+
+ Add an Objective-C object that acts as a delegate for the image analysis overlay view. This is done to correctly
+ position the overlay, but also for a couple of additonal reasons below.
+
+ (-[WKImageAnalysisOverlayViewDelegate dealloc]):
+ (-[WKImageAnalysisOverlayViewDelegate observeValueForKeyPath:ofObject:change:context:]):
+
+ Add logic to steal first responder status away from VKCImageAnalysisOverlayView's internal text selection view
+ and restore it to the web view when there is no longer an active text selection in the overlay. This ensures
+ that key events are only routed to the image analysis overlay view in the case where the overlay actually
+ contains a text selection.
+
+ (-[WKImageAnalysisOverlayViewDelegate firstResponderIsInsideImageOverlay]):
+ (-[WKImageAnalysisOverlayViewDelegate imageAnalysisOverlay:shouldHandleKeyDownEvent:]):
+
+ Never allow the image overlay view to override Escape key handling.
+
+ (-[WKImageAnalysisOverlayViewDelegate contentsRectForImageAnalysisOverlayView:]):
+ (WebKit::WebViewImpl::installImageAnalysisOverlayView):
+ (WebKit::WebViewImpl::uninstallImageAnalysisOverlayView):
+ (WebKit::WebViewImpl::imageAnalysisOverlayViewHasCursorAtPoint const):
+
+ Allow the image analysis overlay view to set the mouse cursor when the mouse is over interactable content.
+
+ * UIProcess/mac/PageClientImplMac.mm:
+ (WebKit::PageClientImpl::setCursor):
+
2022-04-04 Youenn Fablet <[email protected]>
Service-Worker-Navigation-Preload header not being sent when Navigation Preload is enabled.
Modified: trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.h (292315 => 292316)
--- trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.h 2022-04-04 20:54:01 UTC (rev 292315)
+++ trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.h 2022-04-04 21:16:05 UTC (rev 292316)
@@ -61,6 +61,7 @@
OBJC_CLASS WKDOMPasteMenuDelegate;
OBJC_CLASS WKEditorUndoTarget;
OBJC_CLASS WKFullScreenWindowController;
+OBJC_CLASS WKImageAnalysisOverlayViewDelegate;
OBJC_CLASS WKImmediateActionController;
OBJC_CLASS WKMouseTrackingObserver;
OBJC_CLASS WKRevealItemPresenter;
@@ -86,6 +87,9 @@
OBJC_CLASS WKPDFHUDView;
#endif
+OBJC_CLASS VKCImageAnalysis;
+OBJC_CLASS VKCImageAnalysisOverlayView;
+
namespace API {
class HitTestResult;
class Object;
@@ -595,6 +599,13 @@
void computeHasImageAnalysisResults(const URL& imageURL, ShareableBitmap& imageBitmap, ImageAnalysisType, CompletionHandler<void(bool)>&&);
#endif
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ WebCore::FloatRect imageAnalysisInteractionBounds() const { return m_imageAnalysisInteractionBounds; }
+ VKCImageAnalysisOverlayView *imageAnalysisOverlayView() const { return m_imageAnalysisOverlayView.get(); }
+#endif
+
+ bool imageAnalysisOverlayViewHasCursorAtPoint(NSPoint locationInView) const;
+
bool acceptsPreviewPanelControl(QLPreviewPanel *);
void beginPreviewPanelControl(QLPreviewPanel *);
void endPreviewPanelControl(QLPreviewPanel *);
@@ -677,6 +688,11 @@
bool useMediaPlaybackControlsView() const;
bool isRichlyEditableForTouchBar() const;
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ void installImageAnalysisOverlayView(VKCImageAnalysis *);
+ void uninstallImageAnalysisOverlayView();
+#endif
+
bool m_clientWantsMediaPlaybackControlsView { false };
bool m_canCreateTouchBars { false };
bool m_startedListeningToCustomizationEvents { false };
@@ -745,6 +761,7 @@
#if ENABLE(IMAGE_ANALYSIS)
CocoaImageAnalyzer *ensureImageAnalyzer();
+ int32_t processImageAnalyzerRequest(CocoaImageAnalyzerRequest *, CompletionHandler<void(CocoaImageAnalysis *, NSError *)>&&);
#endif
WeakObjCPtr<NSView<WebViewImplDelegate>> m_view;
@@ -891,6 +908,13 @@
RetainPtr<CocoaImageAnalyzer> m_imageAnalyzer;
#endif
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ RetainPtr<VKCImageAnalysisOverlayView> m_imageAnalysisOverlayView;
+ RetainPtr<WKImageAnalysisOverlayViewDelegate> m_imageAnalysisOverlayViewDelegate;
+ uint32_t m_currentImageAnalysisRequestID { 0 };
+ WebCore::FloatRect m_imageAnalysisInteractionBounds;
+#endif
+
#if HAVE(TRANSLATION_UI_SERVICES) && ENABLE(CONTEXT_MENUS)
WeakObjCPtr<NSPopover> m_lastContextMenuTranslationPopover;
#endif
Modified: trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.mm (292315 => 292316)
--- trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.mm 2022-04-04 20:54:01 UTC (rev 292315)
+++ trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.mm 2022-04-04 21:16:05 UTC (rev 292316)
@@ -232,6 +232,15 @@
return m_imageAnalyzer.get();
}
+int32_t WebViewImpl::processImageAnalyzerRequest(CocoaImageAnalyzerRequest *request, CompletionHandler<void(CocoaImageAnalysis *, NSError *)>&& completion)
+{
+ return [ensureImageAnalyzer() processRequest:request progressHandler:nil completionHandler:makeBlockPtr([completion = WTFMove(completion)] (CocoaImageAnalysis *result, NSError *error) mutable {
+ callOnMainRunLoop([completion = WTFMove(completion), result = RetainPtr { result }, error = RetainPtr { error }] () mutable {
+ completion(result.get(), error.get());
+ });
+ }).get()];
+}
+
static RetainPtr<CocoaImageAnalyzerRequest> createImageAnalyzerRequest(CGImageRef image, const URL& imageURL, const URL& pageURL, VKAnalysisTypes types)
{
auto request = createImageAnalyzerRequest(image, types);
@@ -264,12 +273,11 @@
auto request = createImageAnalyzerRequest(cgImage.get(), imageURL, [NSURL _web_URLWithWTFString:m_page->currentURL()], VKAnalysisTypeText);
auto startTime = MonotonicTime::now();
- [ensureImageAnalyzer() processRequest:request.get() progressHandler:nil completionHandler:makeBlockPtr([completion = WTFMove(completion), startTime] (CocoaImageAnalysis *analysis, NSError *) mutable {
- callOnMainRunLoop([completion = WTFMove(completion), result = makeTextRecognitionResult(analysis), startTime] () mutable {
- RELEASE_LOG(Images, "Image analysis completed in %.0f ms (found text? %d)", (MonotonicTime::now() - startTime).milliseconds(), !result.isEmpty());
- completion(WTFMove(result));
- });
- }).get()];
+ processImageAnalyzerRequest(request.get(), [completion = WTFMove(completion), startTime] (CocoaImageAnalysis *analysis, NSError *) mutable {
+ auto result = makeTextRecognitionResult(analysis);
+ RELEASE_LOG(Images, "Image analysis completed in %.0f ms (found text? %d)", (MonotonicTime::now() - startTime).milliseconds(), !result.isEmpty());
+ completion(WTFMove(result));
+ });
}
void WebViewImpl::computeHasImageAnalysisResults(const URL& imageURL, ShareableBitmap& imageBitmap, ImageAnalysisType type, CompletionHandler<void(bool)>&& completion)
@@ -1088,6 +1096,92 @@
@end
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+
+@interface WKImageAnalysisOverlayViewDelegate : NSObject<VKCImageAnalysisOverlayViewDelegate>
+- (instancetype)initWithWebViewImpl:(WebKit::WebViewImpl&)impl;
+@end
+
+@implementation WKImageAnalysisOverlayViewDelegate {
+ WeakPtr<WebKit::WebViewImpl> _impl;
+ __weak VKCImageAnalysisOverlayView *_overlayView;
+ __weak NSResponder *_lastOverlayResponderView;
+}
+
+static void* imageOverlayObservationContext = &imageOverlayObservationContext;
+
+- (instancetype)initWithWebViewImpl:(WebKit::WebViewImpl&)impl
+{
+ if (!(self = [super init]))
+ return nil;
+
+ _impl = impl;
+ _overlayView = impl.imageAnalysisOverlayView();
+ [_overlayView addObserver:self forKeyPath:@"hasActiveTextSelection" options:NSKeyValueObservingOptionOld | NSKeyValueObservingOptionNew context:imageOverlayObservationContext];
+ return self;
+}
+
+- (void)dealloc
+{
+ [_overlayView removeObserver:self forKeyPath:@"hasActiveTextSelection"];
+ [super dealloc];
+}
+
+- (void)observeValueForKeyPath:(NSString *)keyPath ofObject:(id)object change:(NSDictionary *)change context:(void *)context
+{
+ if (context != imageOverlayObservationContext)
+ return;
+
+ BOOL oldHasActiveTextSelection = [change[NSKeyValueChangeOldKey] boolValue];
+ BOOL newHasActiveTextSelection = [change[NSKeyValueChangeNewKey] boolValue];
+ __weak auto webView = _impl ? _impl->view() : nil;
+ auto currentResponder = webView.window.firstResponder;
+ if (oldHasActiveTextSelection && !newHasActiveTextSelection) {
+ if (self.firstResponderIsInsideImageOverlay) {
+ _lastOverlayResponderView = currentResponder;
+ [webView.window makeFirstResponder:webView];
+ }
+ } else if (!oldHasActiveTextSelection && newHasActiveTextSelection) {
+ if (_lastOverlayResponderView && currentResponder != _lastOverlayResponderView)
+ [webView.window makeFirstResponder:_lastOverlayResponderView];
+ }
+}
+
+- (BOOL)firstResponderIsInsideImageOverlay
+{
+ if (!_impl)
+ return NO;
+
+ for (auto view = dynamic_objc_cast<NSView>(_impl->view().window.firstResponder); view; view = view.superview) {
+ if (view == _overlayView)
+ return YES;
+ }
+ return NO;
+}
+
+#pragma mark - VKCImageAnalysisOverlayViewDelegate
+
+- (BOOL)imageAnalysisOverlay:(VKCImageAnalysisOverlayView *)overlayView shouldHandleKeyDownEvent:(NSEvent *)event
+{
+ return ![event.charactersIgnoringModifiers isEqualToString:@"\e"];
+}
+
+- (CGRect)contentsRectForImageAnalysisOverlayView:(VKCImageAnalysisOverlayView *)overlayView
+{
+ if (!_impl)
+ return CGRectMake(0, 0, 1, 1);
+
+ auto unitInteractionRect = _impl->imageAnalysisInteractionBounds();
+ WebCore::FloatRect unobscuredRect = _impl->view().bounds;
+ unitInteractionRect.moveBy(-unobscuredRect.location());
+ unitInteractionRect.scale(1 / unobscuredRect.size());
+ return unitInteractionRect;
+}
+
+@end
+
+#endif // ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+
namespace WebKit {
NSTouchBar *WebViewImpl::makeTouchBar()
@@ -5834,6 +5928,45 @@
#endif // ENABLE(REVEAL)
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+
+void WebViewImpl::installImageAnalysisOverlayView(VKCImageAnalysis *analysis)
+{
+ if (!m_imageAnalysisOverlayView) {
+ m_imageAnalysisOverlayView = adoptNS([PAL::allocVKCImageAnalysisOverlayViewInstance() initWithFrame:[m_view bounds]]);
+ m_imageAnalysisOverlayViewDelegate = adoptNS([[WKImageAnalysisOverlayViewDelegate alloc] initWithWebViewImpl:*this]);
+ [m_imageAnalysisOverlayView setDelegate:m_imageAnalysisOverlayViewDelegate.get()];
+ [m_imageAnalysisOverlayView setActiveInteractionTypes:VKImageAnalysisInteractionTypeTextSelection | VKImageAnalysisInteractionTypeDataDetectors];
+ [m_imageAnalysisOverlayView setWantsAutomaticContentsRectCalculation:NO];
+ }
+
+ [m_imageAnalysisOverlayView setAnalysis:analysis];
+ [m_view addSubview:m_imageAnalysisOverlayView.get()];
+}
+
+void WebViewImpl::uninstallImageAnalysisOverlayView()
+{
+ if (!m_imageAnalysisOverlayView)
+ return;
+
+ [m_imageAnalysisOverlayView removeFromSuperview];
+ m_imageAnalysisOverlayViewDelegate = nil;
+ m_imageAnalysisOverlayView = nil;
+ m_imageAnalysisInteractionBounds = { };
+}
+
+#endif // ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+
+bool WebViewImpl::imageAnalysisOverlayViewHasCursorAtPoint(NSPoint locationInView) const
+{
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+ return [m_imageAnalysisOverlayView interactableItemExistsAtPoint:locationInView];
+#else
+ UNUSED_PARAM(locationInView);
+ return false;
+#endif
+}
+
} // namespace WebKit
#endif // PLATFORM(MAC)
Modified: trunk/Source/WebKit/UIProcess/mac/PageClientImplMac.mm (292315 => 292316)
--- trunk/Source/WebKit/UIProcess/mac/PageClientImplMac.mm 2022-04-04 20:54:01 UTC (rev 292315)
+++ trunk/Source/WebKit/UIProcess/mac/PageClientImplMac.mm 2022-04-04 21:16:05 UTC (rev 292316)
@@ -322,7 +322,8 @@
if (!window)
return;
- if ([window windowNumber] != [NSWindow windowNumberAtPoint:[NSEvent mouseLocation] belowWindowWithWindowNumber:0])
+ auto mouseLocationInScreen = NSEvent.mouseLocation;
+ if (window.windowNumber != [NSWindow windowNumberAtPoint:mouseLocationInScreen belowWindowWithWindowNumber:0])
return;
NSCursor *platformCursor = cursor.platformCursor();
@@ -329,6 +330,9 @@
if ([NSCursor currentCursor] == platformCursor)
return;
+ if (m_impl->imageAnalysisOverlayViewHasCursorAtPoint([m_view convertPoint:mouseLocationInScreen fromView:nil]))
+ return;
+
[platformCursor set];
if (cursor.type() == WebCore::Cursor::Type::None) {
