Diff
Modified: trunk/Source/WebKit/ChangeLog (273777 => 273778)
--- trunk/Source/WebKit/ChangeLog 2021-03-03 01:37:34 UTC (rev 273777)
+++ trunk/Source/WebKit/ChangeLog 2021-03-03 01:56:54 UTC (rev 273778)
@@ -1,3 +1,32 @@
+2021-03-02 Wenson Hsieh <[email protected]>
+
+ [iOS] Image extraction should install image overlay content
+ https://bugs.webkit.org/show_bug.cgi?id=222621
+
+ Reviewed by Tim Horton.
+
+ Take some initial steps towards refactoring image extraction on iOS. See WebKitAdditions for more details.
+
+ * Platform/cocoa/ImageExtractionUtilities.h: Added.
+ * Platform/cocoa/ImageExtractionUtilities.mm: Added.
+
+ Add a new file to contain image-extraction-related functionality; see WebKitAdditions.
+
+ * SourcesCocoa.txt:
+ * UIProcess/WebPageProxy.cpp:
+ (WebKit::WebPageProxy::updateWithImageExtractionResult):
+ * UIProcess/WebPageProxy.h:
+ * UIProcess/ios/WKContentViewInteraction.mm:
+ * WebKit.xcodeproj/project.pbxproj:
+ * WebProcess/WebPage/WebPage.cpp:
+ (WebKit::WebPage::updateWithImageExtractionResult):
+
+ Add IPC plumbing between `WebPageProxy` (in the UI process) and `WebPage` to allow clients to push image
+ extraction results down to a given element on the page.
+
+ * WebProcess/WebPage/WebPage.h:
+ * WebProcess/WebPage/WebPage.messages.in:
+
2021-03-02 Peng Liu <[email protected]>
[ BigSur Debug Wk 2 ] ASSERT NOT REACHED in WebCore::DisplayList::PaintFrameForMedia::apply
Added: trunk/Source/WebKit/Platform/cocoa/ImageExtractionUtilities.h (0 => 273778)
--- trunk/Source/WebKit/Platform/cocoa/ImageExtractionUtilities.h (rev 0)
+++ trunk/Source/WebKit/Platform/cocoa/ImageExtractionUtilities.h 2021-03-03 01:56:54 UTC (rev 273778)
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if USE(APPLE_INTERNAL_SDK)
+#import <WebKitAdditions/ImageExtractionUtilitiesAdditions.h>
+#endif
Added: trunk/Source/WebKit/Platform/cocoa/ImageExtractionUtilities.mm (0 => 273778)
--- trunk/Source/WebKit/Platform/cocoa/ImageExtractionUtilities.mm (rev 0)
+++ trunk/Source/WebKit/Platform/cocoa/ImageExtractionUtilities.mm 2021-03-03 01:56:54 UTC (rev 273778)
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#import "config.h"
+#import "ImageExtractionUtilities.h"
+
+#if USE(APPLE_INTERNAL_SDK)
+#import <WebKitAdditions/ImageExtractionUtilitiesAdditions.mm>
+#endif
Modified: trunk/Source/WebKit/SourcesCocoa.txt (273777 => 273778)
--- trunk/Source/WebKit/SourcesCocoa.txt 2021-03-03 01:37:34 UTC (rev 273777)
+++ trunk/Source/WebKit/SourcesCocoa.txt 2021-03-03 01:56:54 UTC (rev 273778)
@@ -71,6 +71,7 @@
Platform/classifier/cocoa/ResourceLoadStatisticsClassifierCocoa.cpp
Platform/classifier/ResourceLoadStatisticsClassifier.cpp
+Platform/cocoa/ImageExtractionUtilities.mm
Platform/cocoa/LayerHostingContext.mm
Platform/cocoa/PaymentAuthorizationPresenter.mm
Platform/cocoa/PaymentAuthorizationViewController.mm
Modified: trunk/Source/WebKit/UIProcess/WebPageProxy.cpp (273777 => 273778)
--- trunk/Source/WebKit/UIProcess/WebPageProxy.cpp 2021-03-03 01:37:34 UTC (rev 273777)
+++ trunk/Source/WebKit/UIProcess/WebPageProxy.cpp 2021-03-03 01:56:54 UTC (rev 273778)
@@ -8275,6 +8275,16 @@
{
pageClient().requestImageExtraction(imageURL, imageData, WTFMove(completionHandler));
}
+
+void WebPageProxy::updateWithImageExtractionResult(ImageExtractionResult&& results, const ElementContext& context, const FloatPoint& location, CompletionHandler<void(bool textExistsAtLocation)>&& completionHandler)
+{
+ if (!hasRunningProcess()) {
+ completionHandler(false);
+ return;
+ }
+
+ sendWithAsyncReply(Messages::WebPage::UpdateWithImageExtractionResult(WTFMove(results), context, location), WTFMove(completionHandler));
+}
#endif
#if ENABLE(ENCRYPTED_MEDIA)
Modified: trunk/Source/WebKit/UIProcess/WebPageProxy.h (273777 => 273778)
--- trunk/Source/WebKit/UIProcess/WebPageProxy.h 2021-03-03 01:37:34 UTC (rev 273777)
+++ trunk/Source/WebKit/UIProcess/WebPageProxy.h 2021-03-03 01:56:54 UTC (rev 273778)
@@ -1622,6 +1622,7 @@
#if ENABLE(IMAGE_EXTRACTION)
void requestImageExtraction(const URL& imageURL, const ShareableBitmap::Handle& imageData, CompletionHandler<void(WebCore::ImageExtractionResult&&)>&&);
+ void updateWithImageExtractionResult(WebCore::ImageExtractionResult&&, const WebCore::ElementContext&, const WebCore::FloatPoint& location, CompletionHandler<void(bool textExistsAtLocation)>&&);
#endif
#if ENABLE(MEDIA_CONTROLS_CONTEXT_MENUS)
Modified: trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj (273777 => 273778)
--- trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj 2021-03-03 01:37:34 UTC (rev 273777)
+++ trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj 2021-03-03 01:56:54 UTC (rev 273778)
@@ -1997,6 +1997,7 @@
F42D634122A0EFDF00D2FB3A /* WebAutocorrectionData.h in Headers */ = {isa = PBXBuildFile; fileRef = F42D633F22A0EFD300D2FB3A /* WebAutocorrectionData.h */; };
F430E9422247335F005FE053 /* WebsiteMetaViewportPolicy.h in Headers */ = {isa = PBXBuildFile; fileRef = F430E941224732A9005FE053 /* WebsiteMetaViewportPolicy.h */; };
F430E94422473DFF005FE053 /* WebContentMode.h in Headers */ = {isa = PBXBuildFile; fileRef = F430E94322473DB8005FE053 /* WebContentMode.h */; };
+ F4351B9E25EEC84C00D63892 /* ImageExtractionUtilities.h in Headers */ = {isa = PBXBuildFile; fileRef = F4351B9D25EEC84C00D63892 /* ImageExtractionUtilities.h */; };
F438CD1C2241421400DE6DDA /* WKWebpagePreferences.h in Headers */ = {isa = PBXBuildFile; fileRef = F438CD1B224140A600DE6DDA /* WKWebpagePreferences.h */; settings = {ATTRIBUTES = (Public, ); }; };
F438CD1F22414D4000DE6DDA /* WKWebpagePreferencesInternal.h in Headers */ = {isa = PBXBuildFile; fileRef = F438CD1E22414D4000DE6DDA /* WKWebpagePreferencesInternal.h */; };
F438CD212241F69500DE6DDA /* WKWebpagePreferencesPrivate.h in Headers */ = {isa = PBXBuildFile; fileRef = F438CD202241F69500DE6DDA /* WKWebpagePreferencesPrivate.h */; settings = {ATTRIBUTES = (Private, ); }; };
@@ -5876,6 +5877,8 @@
F42D634022A0EFD300D2FB3A /* WebAutocorrectionData.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; name = WebAutocorrectionData.mm; path = ios/WebAutocorrectionData.mm; sourceTree = "<group>"; };
F430E941224732A9005FE053 /* WebsiteMetaViewportPolicy.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WebsiteMetaViewportPolicy.h; sourceTree = "<group>"; };
F430E94322473DB8005FE053 /* WebContentMode.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WebContentMode.h; sourceTree = "<group>"; };
+ F4351B9D25EEC84C00D63892 /* ImageExtractionUtilities.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ImageExtractionUtilities.h; sourceTree = "<group>"; };
+ F4351B9F25EEC87800D63892 /* ImageExtractionUtilities.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = ImageExtractionUtilities.mm; sourceTree = "<group>"; };
F438CD1B224140A600DE6DDA /* WKWebpagePreferences.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WKWebpagePreferences.h; sourceTree = "<group>"; };
F438CD1D22414AD600DE6DDA /* WKWebpagePreferences.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = WKWebpagePreferences.mm; sourceTree = "<group>"; };
F438CD1E22414D4000DE6DDA /* WKWebpagePreferencesInternal.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WKWebpagePreferencesInternal.h; sourceTree = "<group>"; };
@@ -8029,7 +8032,6 @@
51D124851E734AE3002B2820 /* WKHTTPCookieStore.mm */,
51D124861E734AE3002B2820 /* WKHTTPCookieStoreInternal.h */,
DF462E0E23F22F5300EFF35F /* WKHTTPCookieStorePrivate.h */,
- 49917DA6252D649E0050313F /* WKMediaPlaybackState.h */,
1AB40EE31BF677E300BA81BE /* WKMenuItemIdentifiers.mm */,
1AB40EE41BF677E300BA81BE /* WKMenuItemIdentifiersPrivate.h */,
1A5B1C4F1898606F004FCF9B /* WKNavigation.h */,
@@ -8361,6 +8363,8 @@
F4FE0A3A24632B10002631E1 /* CocoaColor.h */,
F4FE0A3C24635667002631E1 /* CocoaFont.h */,
4482734624528F6000A95493 /* CocoaImage.h */,
+ F4351B9D25EEC84C00D63892 /* ImageExtractionUtilities.h */,
+ F4351B9F25EEC87800D63892 /* ImageExtractionUtilities.mm */,
BCE0937614FB128B001138D9 /* LayerHostingContext.h */,
BCE0937514FB128B001138D9 /* LayerHostingContext.mm */,
A1798B3D222D97A2000764BD /* PaymentAuthorizationPresenter.h */,
@@ -11788,6 +11792,7 @@
839A2F321E2067450039057E /* HighPerformanceGraphicsUsageSampler.h in Headers */,
51C0C9741DDD76000032CAD3 /* IconLoadingDelegate.h in Headers */,
51E351CB180F2CCC00E53BE9 /* IDBUtilities.h in Headers */,
+ F4351B9E25EEC84C00D63892 /* ImageExtractionUtilities.h in Headers */,
BCCF6B2512C93E7A008F9C35 /* ImageOptions.h in Headers */,
1A1EC69E1872092100B951F0 /* ImportanceAssertion.h in Headers */,
BC204EE311C83E98008F3375 /* InjectedBundle.h in Headers */,
Modified: trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp (273777 => 273778)
--- trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp 2021-03-03 01:37:34 UTC (rev 273777)
+++ trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp 2021-03-03 01:56:54 UTC (rev 273778)
@@ -7258,6 +7258,20 @@
});
}
+void WebPage::updateWithImageExtractionResult(ImageExtractionResult&& result, const ElementContext& context, const FloatPoint& location, CompletionHandler<void(bool)>&& completionHandler)
+{
+ auto elementToUpdate = elementForContext(context);
+ if (!is<HTMLElement>(elementToUpdate)) {
+ completionHandler(false);
+ return;
+ }
+
+ downcast<HTMLElement>(*elementToUpdate).updateWithImageExtractionResult(WTFMove(result));
+
+ // FIXME: Hit-test with location and return whether or not there is overlay text at the given location.
+ completionHandler(true);
+}
+
#endif // ENABLE(IMAGE_EXTRACTION)
#if ENABLE(MEDIA_CONTROLS_CONTEXT_MENUS)
Modified: trunk/Source/WebKit/WebProcess/WebPage/WebPage.h (273777 => 273778)
--- trunk/Source/WebKit/WebProcess/WebPage/WebPage.h 2021-03-03 01:37:34 UTC (rev 273777)
+++ trunk/Source/WebKit/WebProcess/WebPage/WebPage.h 2021-03-03 01:56:54 UTC (rev 273778)
@@ -1377,6 +1377,7 @@
#if ENABLE(IMAGE_EXTRACTION)
void requestImageExtraction(WebCore::Element&);
+ void updateWithImageExtractionResult(WebCore::ImageExtractionResult&&, const WebCore::ElementContext&, const WebCore::FloatPoint& location, CompletionHandler<void(bool)>&&);
#endif
#if ENABLE(MEDIA_CONTROLS_CONTEXT_MENUS)
Modified: trunk/Source/WebKit/WebProcess/WebPage/WebPage.messages.in (273777 => 273778)
--- trunk/Source/WebKit/WebProcess/WebPage/WebPage.messages.in 2021-03-03 01:37:34 UTC (rev 273777)
+++ trunk/Source/WebKit/WebProcess/WebPage/WebPage.messages.in 2021-03-03 01:56:54 UTC (rev 273778)
@@ -633,4 +633,8 @@
DispatchWheelEventWithoutScrolling(WebKit::WebWheelEvent event) -> (bool handled) Async
LastNavigationWasAppBound() -> (bool wasAppBound) Async
+
+#if ENABLE(IMAGE_EXTRACTION)
+ UpdateWithImageExtractionResult(struct WebCore::ImageExtractionResult result, struct WebCore::ElementContext element, WebCore::FloatPoint location) -> (bool textExistsAtLocation) Async
+#endif
}
