Diff
Modified: trunk/Source/WebCore/ChangeLog (272235 => 272236)
--- trunk/Source/WebCore/ChangeLog 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebCore/ChangeLog 2021-02-02 22:25:32 UTC (rev 272236)
@@ -1,3 +1,31 @@
+2021-02-02 Wenson Hsieh <[email protected]>
+
+ Add scaffolding to request image extraction after a delay when hovering over images
+ https://bugs.webkit.org/show_bug.cgi?id=221226
+
+ Reviewed by Devin Rousso.
+
+ See comments below. There should be no changes in behavior (yet), although this infrastructure will eventually
+ service a new accessibility feature in subsequent patches.
+
+ * page/ChromeClient.h:
+ (WebCore::ChromeClient::requestImageExtraction):
+
+ Adds a new hook to inform the client layer when we should consider requesting image extraction for a hovered
+ element. See WebKit changes for more information.
+
+ * page/EventHandler.cpp:
+ (WebCore::EventHandler::EventHandler):
+ (WebCore::EventHandler::clear):
+ (WebCore::EventHandler::mouseMoved):
+ (WebCore::EventHandler::updateMouseEventTargetNode):
+ (WebCore::EventHandler::imageExtractionTimerFired):
+
+ Add a mechanism to call into the above chrome client hook when the user has hovered their mouse cursor over an
+ image (e.g. `img` or `picture` element) for at least 250 ms without moving.
+
+ * page/EventHandler.h:
+
2021-02-02 Darin Adler <[email protected]>
Null check renderers consistently in StyleImage code
Modified: trunk/Source/WebCore/page/ChromeClient.h (272235 => 272236)
--- trunk/Source/WebCore/page/ChromeClient.h 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebCore/page/ChromeClient.h 2021-02-02 22:25:32 UTC (rev 272236)
@@ -554,6 +554,10 @@
virtual void changeUniversalAccessZoomFocus(const IntRect&, const IntRect&) { }
#endif
+#if ENABLE(IMAGE_EXTRACTION)
+ virtual void requestImageExtraction(Element&) { }
+#endif
+
protected:
virtual ~ChromeClient() = default;
};
Modified: trunk/Source/WebCore/page/EventHandler.cpp (272235 => 272236)
--- trunk/Source/WebCore/page/EventHandler.cpp 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebCore/page/EventHandler.cpp 2021-02-02 22:25:32 UTC (rev 272236)
@@ -321,6 +321,9 @@
EventHandler::EventHandler(Frame& frame)
: m_frame(frame)
, m_hoverTimer(*this, &EventHandler::hoverTimerFired)
+#if ENABLE(IMAGE_EXTRACTION)
+ , m_imageExtractionTimer(*this, &EventHandler::imageExtractionTimerFired, 250_ms)
+#endif
, m_autoscrollController(makeUnique<AutoscrollController>())
#if !ENABLE(IOS_TOUCH_EVENTS)
, m_fakeMouseMoveEventTimer(*this, &EventHandler::fakeMouseMoveEventTimerFired)
@@ -361,6 +364,9 @@
#if ENABLE(CURSOR_VISIBILITY)
cancelAutoHideCursorTimer();
#endif
+#if ENABLE(IMAGE_EXTRACTION)
+ m_imageExtractionTimer.stop();
+#endif
m_resizeLayer = nullptr;
m_elementUnderMouse = nullptr;
m_lastElementUnderMouse = nullptr;
@@ -1872,6 +1878,12 @@
hitTestResult.setToNonUserAgentShadowAncestor();
page->chrome().mouseDidMoveOverElement(hitTestResult, event.modifierFlags());
+
+#if ENABLE(IMAGE_EXTRACTION)
+ if (event.syntheticClickType() == NoTap && m_imageExtractionTimer.isActive())
+ m_imageExtractionTimer.restart();
+#endif
+
return result;
}
@@ -2499,6 +2511,15 @@
m_elementUnderMouse = targetElement;
+#if ENABLE(IMAGE_EXTRACTION)
+ if (platformMouseEvent.syntheticClickType() == NoTap) {
+ if (m_elementUnderMouse && is<RenderImage>(m_elementUnderMouse->renderer()))
+ m_imageExtractionTimer.restart();
+ else
+ m_imageExtractionTimer.stop();
+ }
+#endif
+
ASSERT_IMPLIES(m_elementUnderMouse, &m_elementUnderMouse->document() == m_frame.document());
ASSERT_IMPLIES(m_lastElementUnderMouse, &m_lastElementUnderMouse->document() == m_frame.document());
@@ -2554,8 +2575,12 @@
}
// Event handling may have moved the element to a different document.
- if (m_elementUnderMouse && &m_elementUnderMouse->document() != m_frame.document())
+ if (m_elementUnderMouse && &m_elementUnderMouse->document() != m_frame.document()) {
+#if ENABLE(IMAGE_EXTRACTION)
+ m_imageExtractionTimer.stop();
+#endif
m_elementUnderMouse = nullptr;
+ }
m_lastElementUnderMouse = m_elementUnderMouse;
}
@@ -3322,6 +3347,19 @@
}
}
+#if ENABLE(IMAGE_EXTRACTION)
+
+void EventHandler::imageExtractionTimerFired()
+{
+ if (!m_elementUnderMouse || !is<RenderImage>(m_elementUnderMouse->renderer()))
+ return;
+
+ if (auto* page = m_frame.page())
+ page->chrome().client().requestImageExtraction(*m_elementUnderMouse);
+}
+
+#endif // ENABLE(IMAGE_EXTRACTION)
+
bool EventHandler::handleAccessKey(const PlatformKeyboardEvent& event)
{
// FIXME: Ignoring the state of Shift key is what neither IE nor Firefox do.
Modified: trunk/Source/WebCore/page/EventHandler.h (272235 => 272236)
--- trunk/Source/WebCore/page/EventHandler.h 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebCore/page/EventHandler.h 2021-02-02 22:25:32 UTC (rev 272236)
@@ -387,6 +387,10 @@
void hoverTimerFired();
+#if ENABLE(IMAGE_EXTRACTION)
+ void imageExtractionTimerFired();
+#endif
+
bool logicalScrollOverflow(ScrollLogicalDirection, ScrollGranularity, Node* startingNode = nullptr);
bool shouldSwapScrollDirection(const HitTestResult&, const PlatformWheelEvent&) const;
@@ -530,6 +534,9 @@
Frame& m_frame;
RefPtr<Node> m_mousePressNode;
Timer m_hoverTimer;
+#if ENABLE(IMAGE_EXTRACTION)
+ DeferrableOneShotTimer m_imageExtractionTimer;
+#endif
std::unique_ptr<AutoscrollController> m_autoscrollController;
RenderLayer* m_resizeLayer { nullptr };
Modified: trunk/Source/WebKit/ChangeLog (272235 => 272236)
--- trunk/Source/WebKit/ChangeLog 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/ChangeLog 2021-02-02 22:25:32 UTC (rev 272236)
@@ -1,3 +1,43 @@
+2021-02-02 Wenson Hsieh <[email protected]>
+
+ Add scaffolding to request image extraction after a delay when hovering over images
+ https://bugs.webkit.org/show_bug.cgi?id=221226
+
+ Reviewed by Devin Rousso.
+
+ Work towards allowing clients to extract image data when hovering over rendered image elements, for new
+ accessibility-related features. See changes below for more details.
+
+ * UIProcess/Cocoa/WebViewImpl.h:
+ * UIProcess/Cocoa/WebViewImpl.mm:
+ (WebKit::WebViewImpl::requestImageExtraction):
+
+ Leave this method stub unimplemented for the time being.
+
+ * UIProcess/PageClient.h:
+ (WebKit::PageClient::requestImageExtraction):
+ * UIProcess/WebPageProxy.cpp:
+ (WebKit::WebPageProxy::requestImageExtraction):
+ * UIProcess/WebPageProxy.h:
+ * UIProcess/WebPageProxy.messages.in:
+ * UIProcess/mac/PageClientImplMac.h:
+ * UIProcess/mac/PageClientImplMac.mm:
+ (WebKit::PageClientImpl::requestImageExtraction):
+
+ Add plumbing to allow the web process to asynchronously request image extraction data from the UI process.
+
+ * WebProcess/WebCoreSupport/WebChromeClient.cpp:
+ (WebKit::WebChromeClient::requestImageExtraction):
+ * WebProcess/WebCoreSupport/WebChromeClient.h:
+ * WebProcess/WebPage/WebPage.cpp:
+ (WebKit::WebPage::didCommitLoad):
+ (WebKit::WebPage::requestImageExtraction):
+ * WebProcess/WebPage/WebPage.h:
+
+ Add a weak set of elements that we've attempted to extract, so that we won't end up extracting elements more
+ than once. Although this set this currently only cleared upon navigation, this should eventually be updated when
+ (for instance) an element's image source changes, such that we can re-extract the new image data if needed.
+
2021-02-02 Kate Cheney <[email protected]>
REGRESSION(r271744): Broke Microsoft live login in internal builds
Modified: trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.h (272235 => 272236)
--- trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.h 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.h 2021-02-02 22:25:32 UTC (rev 272236)
@@ -89,6 +89,10 @@
namespace WebCore {
struct ShareDataWithParsedURL;
+
+#if ENABLE(IMAGE_EXTRACTION)
+struct ImageExtractionResult;
+#endif
}
@protocol WebViewImplDelegate
@@ -575,6 +579,10 @@
void forceRequestCandidatesForTesting();
bool shouldRequestCandidates() const;
+#if ENABLE(IMAGE_EXTRACTION)
+ void requestImageExtraction(const ShareableBitmap::Handle&, CompletionHandler<void(WebCore::ImageExtractionResult&&)>&&);
+#endif
+
bool windowIsFrontWindowUnderMouse(NSEvent *);
void setRequiresUserActionForEditingControlsManager(bool requiresUserActionForEditingControlsManager) { m_requiresUserActionForEditingControlsManager = requiresUserActionForEditingControlsManager; }
Modified: trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.mm (272235 => 272236)
--- trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.mm 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/UIProcess/Cocoa/WebViewImpl.mm 2021-02-02 22:25:32 UTC (rev 272236)
@@ -134,6 +134,10 @@
#import <wtf/cocoa/VectorCocoa.h>
#import <wtf/text/StringConcatenate.h>
+#if ENABLE(IMAGE_EXTRACTION)
+#import <WebCore/ImageExtractionResult.h>
+#endif
+
#if HAVE(TOUCH_BAR) && ENABLE(WEB_PLAYBACK_CONTROLS_MANAGER)
SOFT_LINK_FRAMEWORK(AVKit)
SOFT_LINK_CLASS(AVKit, AVTouchBarPlaybackControlsProvider)
@@ -5582,6 +5586,16 @@
return false;
}
+#if ENABLE(IMAGE_EXTRACTION)
+
+void WebViewImpl::requestImageExtraction(const ShareableBitmap::Handle& imageData, CompletionHandler<void(ImageExtractionResult&&)>&& completion)
+{
+ UNUSED_PARAM(imageData);
+ completion({ });
+}
+
+#endif // ENABLE(IMAGE_EXTRACTION)
+
} // namespace WebKit
#endif // PLATFORM(MAC)
Modified: trunk/Source/WebKit/UIProcess/PageClient.h (272235 => 272236)
--- trunk/Source/WebKit/UIProcess/PageClient.h 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/UIProcess/PageClient.h 2021-02-02 22:25:32 UTC (rev 272236)
@@ -59,6 +59,10 @@
#include <WebCore/InspectorOverlay.h>
#endif
+#if ENABLE(IMAGE_EXTRACTION)
+#include <WebCore/ImageExtractionResult.h>
+#endif
+
OBJC_CLASS CALayer;
OBJC_CLASS NSFileWrapper;
OBJC_CLASS NSMenu;
@@ -507,6 +511,10 @@
virtual bool hasSafeBrowsingWarning() const { return false; }
virtual void setMouseEventPolicy(WebCore::MouseEventPolicy) { }
+
+#if ENABLE(IMAGE_EXTRACTION)
+ virtual void requestImageExtraction(const ShareableBitmap::Handle&, CompletionHandler<void(WebCore::ImageExtractionResult&&)>&& completion) { completion({ }); }
+#endif
#if PLATFORM(MAC)
virtual void didPerformImmediateActionHitTest(const WebHitTestResultData&, bool contentPreventsDefault, API::Object*) = 0;
Modified: trunk/Source/WebKit/UIProcess/WebPageProxy.cpp (272235 => 272236)
--- trunk/Source/WebKit/UIProcess/WebPageProxy.cpp 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/UIProcess/WebPageProxy.cpp 2021-02-02 22:25:32 UTC (rev 272236)
@@ -8301,6 +8301,15 @@
}
#endif
+#if ENABLE(IMAGE_EXTRACTION)
+
+void WebPageProxy::requestImageExtraction(const ShareableBitmap::Handle& imageData, CompletionHandler<void(WebCore::ImageExtractionResult&&)>&& completionHandler)
+{
+ pageClient().requestImageExtraction(imageData, WTFMove(completionHandler));
+}
+
+#endif
+
void WebPageProxy::requestNotificationPermission(uint64_t requestID, const String& originString)
{
if (!isRequestIDValid(requestID))
Modified: trunk/Source/WebKit/UIProcess/WebPageProxy.h (272235 => 272236)
--- trunk/Source/WebKit/UIProcess/WebPageProxy.h 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/UIProcess/WebPageProxy.h 2021-02-02 22:25:32 UTC (rev 272236)
@@ -271,6 +271,10 @@
struct ViewportAttributes;
struct WindowFeatures;
+#if ENABLE(IMAGE_EXTRACTION)
+struct ImageExtractionResult;
+#endif
+
template<typename> class RectEdges;
using FloatBoxExtent = RectEdges<float>;
}
@@ -1611,6 +1615,10 @@
void shouldAllowDeviceOrientationAndMotionAccess(WebCore::FrameIdentifier, FrameInfoData&&, bool mayPrompt, CompletionHandler<void(WebCore::DeviceOrientationOrMotionPermissionState)>&&);
#endif
+#if ENABLE(IMAGE_EXTRACTION)
+ void requestImageExtraction(const ShareableBitmap::Handle&, CompletionHandler<void(WebCore::ImageExtractionResult&&)>&&);
+#endif
+
static WebPageProxy* nonEphemeralWebPageProxy();
#if ENABLE(ATTACHMENT_ELEMENT)
Modified: trunk/Source/WebKit/UIProcess/WebPageProxy.messages.in (272235 => 272236)
--- trunk/Source/WebKit/UIProcess/WebPageProxy.messages.in 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/UIProcess/WebPageProxy.messages.in 2021-02-02 22:25:32 UTC (rev 272236)
@@ -239,6 +239,10 @@
CompositionWasCanceled()
SetHasHadSelectionChangesFromUserInteraction(bool hasHadUserSelectionChanges)
+#if ENABLE(IMAGE_EXTRACTION)
+ RequestImageExtraction(WebKit::ShareableBitmap::Handle imageData) -> (struct WebCore::ImageExtractionResult result) Async
+#endif
+
#if HAVE(TOUCH_BAR)
SetIsTouchBarUpdateSupressedForHiddenContentEditable(bool isTouchBarUpdateSupressed)
SetIsNeverRichlyEditableForTouchBar(bool isNeverRichlyEditable)
Modified: trunk/Source/WebKit/UIProcess/mac/PageClientImplMac.h (272235 => 272236)
--- trunk/Source/WebKit/UIProcess/mac/PageClientImplMac.h 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/UIProcess/mac/PageClientImplMac.h 2021-02-02 22:25:32 UTC (rev 272236)
@@ -130,6 +130,10 @@
void doneWithKeyEvent(const NativeWebKeyboardEvent&, bool wasEventHandled) override;
+#if ENABLE(IMAGE_EXTRACTION)
+ void requestImageExtraction(const ShareableBitmap::Handle&, CompletionHandler<void(WebCore::ImageExtractionResult&&)>&&) override;
+#endif
+
RefPtr<WebPopupMenuProxy> createPopupMenuProxy(WebPageProxy&) override;
#if ENABLE(CONTEXT_MENUS)
Ref<WebContextMenuProxy> createContextMenuProxy(WebPageProxy&, ContextMenuContextData&&, const UserData&) override;
Modified: trunk/Source/WebKit/UIProcess/mac/PageClientImplMac.mm (272235 => 272236)
--- trunk/Source/WebKit/UIProcess/mac/PageClientImplMac.mm 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/UIProcess/mac/PageClientImplMac.mm 2021-02-02 22:25:32 UTC (rev 272236)
@@ -481,6 +481,15 @@
m_impl->doneWithKeyEvent(event.nativeEvent(), eventWasHandled);
}
+#if ENABLE(IMAGE_EXTRACTION)
+
+void PageClientImpl::requestImageExtraction(const ShareableBitmap::Handle& imageData, CompletionHandler<void(ImageExtractionResult&&)>&& completion)
+{
+ m_impl->requestImageExtraction(imageData, WTFMove(completion));
+}
+
+#endif
+
RefPtr<WebPopupMenuProxy> PageClientImpl::createPopupMenuProxy(WebPageProxy& page)
{
return WebPopupMenuProxyMac::create(m_view, page);
Modified: trunk/Source/WebKit/WebProcess/WebCoreSupport/WebChromeClient.cpp (272235 => 272236)
--- trunk/Source/WebKit/WebProcess/WebCoreSupport/WebChromeClient.cpp 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/WebProcess/WebCoreSupport/WebChromeClient.cpp 2021-02-02 22:25:32 UTC (rev 272236)
@@ -1431,4 +1431,13 @@
}
#endif
+#if ENABLE(IMAGE_EXTRACTION)
+
+void WebChromeClient::requestImageExtraction(Element& element)
+{
+ m_page.requestImageExtraction(element);
+}
+
+#endif
+
} // namespace WebKit
Modified: trunk/Source/WebKit/WebProcess/WebCoreSupport/WebChromeClient.h (272235 => 272236)
--- trunk/Source/WebKit/WebProcess/WebCoreSupport/WebChromeClient.h 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/WebProcess/WebCoreSupport/WebChromeClient.h 2021-02-02 22:25:32 UTC (rev 272236)
@@ -414,6 +414,10 @@
void changeUniversalAccessZoomFocus(const WebCore::IntRect&, const WebCore::IntRect&) final;
#endif
+#if ENABLE(IMAGE_EXTRACTION)
+ void requestImageExtraction(WebCore::Element&) final;
+#endif
+
mutable bool m_cachedMainFrameHasHorizontalScrollbar { false };
mutable bool m_cachedMainFrameHasVerticalScrollbar { false };
Modified: trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp (272235 => 272236)
--- trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp 2021-02-02 22:25:32 UTC (rev 272236)
@@ -347,6 +347,10 @@
#endif
#endif
+#if ENABLE(IMAGE_EXTRACTION)
+#include <WebCore/ImageExtractionResult.h>
+#endif
+
namespace WebKit {
using namespace JSC;
using namespace WebCore;
@@ -6062,6 +6066,10 @@
ASSERT(!frame->coreFrame()->loader().stateMachine().creatingInitialEmptyDocument());
unfreezeLayerTree(LayerTreeFreezeReason::ProcessSwap);
+#if ENABLE(IMAGE_EXTRACTION)
+ m_elementsWithExtractedImages.clear();
+#endif
+
#if ENABLE(RESOURCE_LOAD_STATISTICS)
clearLoadedSubresourceDomains();
#endif
@@ -7119,6 +7127,36 @@
}
#endif // ENABLE(MEDIA_USAGE)
+#if ENABLE(IMAGE_EXTRACTION)
+
+void WebPage::requestImageExtraction(WebCore::Element& element)
+{
+ if (m_elementsWithExtractedImages.contains(element))
+ return;
+
+ auto* renderImage = element.renderer();
+ if (!is<RenderImage>(renderImage))
+ return;
+
+ m_elementsWithExtractedImages.add(element);
+
+ auto bitmap = shareableBitmap(downcast<RenderImage>(*renderImage));
+ if (!bitmap)
+ return;
+
+ ShareableBitmap::Handle bitmapHandle;
+ bitmap->createHandle(bitmapHandle);
+ if (bitmapHandle.isNull())
+ return;
+
+ sendWithAsyncReply(Messages::WebPageProxy::RequestImageExtraction(WTFMove(bitmapHandle)), [weakElement = makeWeakPtr(element)] (ImageExtractionResult&& result) {
+ UNUSED_PARAM(result);
+ UNUSED_PARAM(weakElement);
+ });
+}
+
+#endif // ENABLE(IMAGE_EXTRACTION)
+
#if !PLATFORM(IOS_FAMILY)
void WebPage::animationDidFinishForElement(const WebCore::Element&)
Modified: trunk/Source/WebKit/WebProcess/WebPage/WebPage.h (272235 => 272236)
--- trunk/Source/WebKit/WebProcess/WebPage/WebPage.h 2021-02-02 22:00:32 UTC (rev 272235)
+++ trunk/Source/WebKit/WebProcess/WebPage/WebPage.h 2021-02-02 22:25:32 UTC (rev 272236)
@@ -1367,6 +1367,10 @@
void removeMediaUsageManagerSession(WebCore::MediaSessionIdentifier);
#endif
+#if ENABLE(IMAGE_EXTRACTION)
+ void requestImageExtraction(WebCore::Element&);
+#endif
+
#if PLATFORM(WIN)
uint64_t nativeWindowHandle() { return m_nativeWindowHandle; }
#endif
@@ -2212,6 +2216,10 @@
#if ENABLE(GPU_PROCESS)
std::unique_ptr<RemoteRenderingBackendProxy> m_remoteRenderingBackendProxy;
#endif
+
+#if ENABLE(IMAGE_EXTRACTION)
+ WeakHashSet<WebCore::Element> m_elementsWithExtractedImages;
+#endif
};
#if !PLATFORM(IOS_FAMILY)
