Diff
Modified: trunk/Source/WebCore/ChangeLog (292251 => 292252)
--- trunk/Source/WebCore/ChangeLog 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebCore/ChangeLog 2022-04-02 02:09:56 UTC (rev 292252)
@@ -1,3 +1,18 @@
+2022-04-01 Wenson Hsieh <[email protected]>
+
+ [iOS] [WK2] Add plumbing for extracting video frames in element fullscreen
+ https://bugs.webkit.org/show_bug.cgi?id=238607
+ rdar://91102888
+
+ Reviewed by Eric Carlson.
+
+ Add a helper function on HTMLMediaElement to return the identifier of its MediaPlayer. See WebKit ChangeLog for
+ more information.
+
+ * html/HTMLMediaElement.cpp:
+ (WebCore::HTMLMediaElement::playerIdentifier const):
+ * html/HTMLMediaElement.h:
+
2022-04-01 Simon Fraser <[email protected]>
Hoist the IOSurfacePool out of IOSurface
Modified: trunk/Source/WebCore/html/HTMLMediaElement.cpp (292251 => 292252)
--- trunk/Source/WebCore/html/HTMLMediaElement.cpp 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebCore/html/HTMLMediaElement.cpp 2022-04-02 02:09:56 UTC (rev 292252)
@@ -621,6 +621,11 @@
ThreadableBlobRegistry::unregisterBlobURL(m_blobURLForReading);
}
+std::optional<MediaPlayerIdentifier> HTMLMediaElement::playerIdentifier() const
+{
+ return m_player ? std::optional { m_player->identifier() } : std::nullopt;
+}
+
RefPtr<HTMLMediaElement> HTMLMediaElement::bestMediaElementForRemoteControls(MediaElementSession::PlaybackControlsPurpose purpose, const Document* document)
{
Vector<MediaElementSessionInfo> candidateSessions;
Modified: trunk/Source/WebCore/html/HTMLMediaElement.h (292251 => 292252)
--- trunk/Source/WebCore/html/HTMLMediaElement.h 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebCore/html/HTMLMediaElement.h 2022-04-02 02:09:56 UTC (rev 292252)
@@ -155,6 +155,8 @@
using HTMLElement::weakPtrFactory;
RefPtr<MediaPlayer> player() const { return m_player; }
+ WEBCORE_EXPORT std::optional<MediaPlayerIdentifier> playerIdentifier() const;
+
bool supportsAcceleratedRendering() const { return m_cachedSupportsAcceleratedRendering; }
virtual bool isVideo() const { return false; }
Modified: trunk/Source/WebKit/ChangeLog (292251 => 292252)
--- trunk/Source/WebKit/ChangeLog 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/ChangeLog 2022-04-02 02:09:56 UTC (rev 292252)
@@ -1,3 +1,51 @@
+2022-04-01 Wenson Hsieh <[email protected]>
+
+ [iOS] [WK2] Add plumbing for extracting video frames in element fullscreen
+ https://bugs.webkit.org/show_bug.cgi?id=238607
+ rdar://91102888
+
+ Reviewed by Eric Carlson.
+
+ Add WebKit2 client layer plumbing for triggering video extraction for the prominent video in element fullscreen.
+ See below for more details.
+
+ * UIProcess/PageClient.h:
+ (WebKit::PageClient::beginElementFullscreenVideoExtraction):
+ (WebKit::PageClient::cancelElementFullscreenVideoExtraction):
+
+ Add new PageClient hooks to start and cancel video element extraction in element fullscreen. We only implement
+ these on iOS, for now.
+
+ * UIProcess/WebPageProxy.cpp:
+ (WebKit::WebPageProxy::extractVideoInElementFullScreen):
+ (WebKit::WebPageProxy::cancelVideoExtractionInElementFullScreen):
+ * UIProcess/WebPageProxy.h:
+ * UIProcess/WebPageProxy.messages.in:
+
+ Add IPC plumbing from WebPage to WebPageProxy for starting and canceling video element extraction.
+
+ * UIProcess/ios/PageClientImplIOS.h:
+ * UIProcess/ios/PageClientImplIOS.mm:
+ (WebKit::PageClientImpl::beginElementFullscreenVideoExtraction):
+ (WebKit::PageClientImpl::cancelElementFullscreenVideoExtraction):
+ * UIProcess/ios/WKContentViewInteraction.h:
+ * UIProcess/ios/WKContentViewInteraction.mm:
+ (-[WKContentView beginElementFullscreenVideoExtraction:bounds:]):
+ (-[WKContentView cancelElementFullscreenVideoExtraction]):
+
+ Add method stubs for non-internal builds.
+
+ * WebProcess/FullScreen/WebFullScreenManager.cpp:
+ (WebKit::WebFullScreenManager::mainVideoElementExtractionTimerFired):
+ (WebKit::WebFullScreenManager::endMainVideoElementExtractionIfNeeded):
+
+ Remove the FIXMEs and call into the new helper methods on WebPage to start and cancel video element extraction.
+
+ * WebProcess/WebPage/WebPage.cpp:
+ (WebKit::WebPage::extractVideoInElementFullScreen):
+ (WebKit::WebPage::cancelVideoExtractionInElementFullScreen):
+ * WebProcess/WebPage/WebPage.h:
+
2022-04-01 Simon Fraser <[email protected]>
Hoist the IOSurfacePool out of IOSurface
Modified: trunk/Source/WebKit/UIProcess/PageClient.h (292251 => 292252)
--- trunk/Source/WebKit/UIProcess/PageClient.h 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/UIProcess/PageClient.h 2022-04-02 02:09:56 UTC (rev 292252)
@@ -509,6 +509,9 @@
#endif
virtual bool isFullscreenVideoExtractionEnabled() const { return false; }
+ virtual void beginElementFullscreenVideoExtraction(const ShareableBitmap::Handle&, WebCore::FloatRect) { }
+ virtual void cancelElementFullscreenVideoExtraction() { }
+
// Auxiliary Client Creation
#if ENABLE(FULLSCREEN_API)
virtual WebFullScreenManagerProxyClient& fullScreenManagerProxyClient() = 0;
Modified: trunk/Source/WebKit/UIProcess/WebPageProxy.cpp (292251 => 292252)
--- trunk/Source/WebKit/UIProcess/WebPageProxy.cpp 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/UIProcess/WebPageProxy.cpp 2022-04-02 02:09:56 UTC (rev 292252)
@@ -11309,6 +11309,40 @@
m_uiClient->decidePolicyForModalContainer(types, WTFMove(completion));
}
+void WebPageProxy::extractVideoInElementFullScreen(MediaPlayerIdentifier identifier, FloatRect bounds)
+{
+ if (!pageClient().isFullscreenVideoExtractionEnabled())
+ return;
+
+#if ENABLE(GPU_PROCESS)
+ RefPtr gpuProcess = GPUProcessProxy::singletonIfCreated();
+ if (!gpuProcess)
+ return;
+
+ m_hasPendingElementFullScreenVideoExtraction = true;
+ gpuProcess->requestBitmapImageForCurrentTime(m_process->coreProcessIdentifier(), identifier, [weakThis = WeakPtr { *this }, bounds](auto& bitmapHandle) {
+ RefPtr protectedThis = weakThis.get();
+ if (!protectedThis || !protectedThis->m_hasPendingElementFullScreenVideoExtraction)
+ return;
+
+ protectedThis->pageClient().beginElementFullscreenVideoExtraction(bitmapHandle, bounds);
+ protectedThis->m_hasPendingElementFullScreenVideoExtraction = false;
+ });
+#else
+ UNUSED_PARAM(identifier);
+ UNUSED_PARAM(bounds);
+#endif
+}
+
+void WebPageProxy::cancelVideoExtractionInElementFullScreen()
+{
+ if (!pageClient().isFullscreenVideoExtractionEnabled())
+ return;
+
+ m_hasPendingElementFullScreenVideoExtraction = false;
+ pageClient().cancelElementFullscreenVideoExtraction();
+}
+
} // namespace WebKit
#undef WEBPAGEPROXY_RELEASE_LOG
Modified: trunk/Source/WebKit/UIProcess/WebPageProxy.h (292251 => 292252)
--- trunk/Source/WebKit/UIProcess/WebPageProxy.h 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/UIProcess/WebPageProxy.h 2022-04-02 02:09:56 UTC (rev 292252)
@@ -2093,6 +2093,9 @@
}
#endif
+ void extractVideoInElementFullScreen(WebCore::MediaPlayerIdentifier, WebCore::FloatRect videoBounds);
+ void cancelVideoExtractionInElementFullScreen();
+
private:
WebPageProxy(PageClient&, WebProcessProxy&, Ref<API::PageConfiguration>&&);
void platformInitialize();
@@ -3240,6 +3243,7 @@
std::optional<PlaybackSessionContextIdentifier> m_currentFullscreenVideoSessionIdentifier;
RunLoop::Timer<WebPageProxy> m_fullscreenVideoExtractionTimer;
#endif
+ bool m_hasPendingElementFullScreenVideoExtraction { false };
#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
enum class CroppedImageOverlayState : uint8_t {
Modified: trunk/Source/WebKit/UIProcess/WebPageProxy.messages.in (292251 => 292252)
--- trunk/Source/WebKit/UIProcess/WebPageProxy.messages.in 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/UIProcess/WebPageProxy.messages.in 2022-04-02 02:09:56 UTC (rev 292252)
@@ -378,6 +378,9 @@
DictationAlternatives(WebCore::DictationContext dictationContext) -> (Vector<String> alternatives) Synchronous
#endif
+ ExtractVideoInElementFullScreen(WebCore::MediaPlayerIdentifier identifier, WebCore::FloatRect videoBounds)
+ CancelVideoExtractionInElementFullScreen()
+
#if PLATFORM(IOS_FAMILY)
CouldNotRestorePageState()
RestorePageState(std::optional<WebCore::FloatPoint> scrollPosition, WebCore::FloatPoint scrollOrigin, WebCore::RectEdges<float> obscuredInsetsOnSave, double scale)
Modified: trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.h (292251 => 292252)
--- trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.h 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.h 2022-04-02 02:09:56 UTC (rev 292252)
@@ -307,6 +307,9 @@
void cancelFullscreenVideoExtraction(AVPlayerViewController *) final;
bool isFullscreenVideoExtractionEnabled() const final;
+ void beginElementFullscreenVideoExtraction(const ShareableBitmap::Handle&, WebCore::FloatRect) final;
+ void cancelElementFullscreenVideoExtraction() final;
+
WeakObjCPtr<WKContentView> m_contentView;
RetainPtr<WKEditorUndoTarget> m_undoTarget;
};
Modified: trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.mm (292251 => 292252)
--- trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.mm 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/UIProcess/ios/PageClientImplIOS.mm 2022-04-02 02:09:56 UTC (rev 292252)
@@ -1060,6 +1060,16 @@
return [m_contentView isFullscreenVideoExtractionEnabled];
}
+void PageClientImpl::beginElementFullscreenVideoExtraction(const ShareableBitmap::Handle& bitmapHandle, FloatRect bounds)
+{
+ [m_contentView beginElementFullscreenVideoExtraction:bitmapHandle bounds:bounds];
+}
+
+void PageClientImpl::cancelElementFullscreenVideoExtraction()
+{
+ [m_contentView cancelElementFullscreenVideoExtraction];
+}
+
} // namespace WebKit
#endif // PLATFORM(IOS_FAMILY)
Modified: trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.h (292251 => 292252)
--- trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.h 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.h 2022-04-02 02:09:56 UTC (rev 292252)
@@ -804,6 +804,9 @@
- (void)cancelFullscreenVideoExtraction:(AVPlayerViewController *)controller;
@property (nonatomic, readonly) BOOL isFullscreenVideoExtractionEnabled;
+- (void)beginElementFullscreenVideoExtraction:(const WebKit::ShareableBitmap::Handle&)bitmapHandle bounds:(WebCore::FloatRect)bounds;
+- (void)cancelElementFullscreenVideoExtraction;
+
@end
@interface WKContentView (WKTesting)
Modified: trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm (292251 => 292252)
--- trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/UIProcess/ios/WKContentViewInteraction.mm 2022-04-02 02:09:56 UTC (rev 292252)
@@ -11048,6 +11048,14 @@
return NO;
}
+- (void)beginElementFullscreenVideoExtraction:(const WebKit::ShareableBitmap::Handle&)bitmapHandle bounds:(WebCore::FloatRect)bounds
+{
+}
+
+- (void)cancelElementFullscreenVideoExtraction
+{
+}
+
#endif
@end
Modified: trunk/Source/WebKit/WebProcess/FullScreen/WebFullScreenManager.cpp (292251 => 292252)
--- trunk/Source/WebKit/WebProcess/FullScreen/WebFullScreenManager.cpp 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/WebProcess/FullScreen/WebFullScreenManager.cpp 2022-04-02 02:09:56 UTC (rev 292252)
@@ -361,8 +361,11 @@
if (!m_mainVideoElement)
return;
+ if (m_isExtractingMainVideoElement)
+ m_page->cancelVideoExtractionInElementFullScreen();
+
m_isExtractingMainVideoElement = true;
- // FIXME: Begin main video extraction in for element fullscreen.
+ m_page->extractVideoInElementFullScreen(*m_mainVideoElement);
}
void WebFullScreenManager::scheduleMainVideoElementExtraction()
@@ -375,7 +378,7 @@
m_mainVideoElementExtractionTimer.stop();
if (m_isExtractingMainVideoElement) {
- // FIXME: Cancel the current video extraction session.
+ m_page->cancelVideoExtractionInElementFullScreen();
m_isExtractingMainVideoElement = false;
}
}
Modified: trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp (292251 => 292252)
--- trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/WebProcess/WebPage/WebPage.cpp 2022-04-02 02:09:56 UTC (rev 292252)
@@ -236,6 +236,7 @@
#include <WebCore/RenderLayer.h>
#include <WebCore/RenderTheme.h>
#include <WebCore/RenderTreeAsText.h>
+#include <WebCore/RenderVideo.h>
#include <WebCore/RenderView.h>
#include <WebCore/ResourceLoadStatistics.h>
#include <WebCore/ResourceRequest.h>
@@ -8086,6 +8087,32 @@
mainFrameView()->setScrollPosition(IntPoint(targetRect.minXMinYCorner()));
}
+void WebPage::extractVideoInElementFullScreen(const HTMLVideoElement& element)
+{
+ RefPtr view = element.document().view();
+ if (!view)
+ return;
+
+ auto mediaPlayerIdentifier = valueOrDefault(element.playerIdentifier());
+ if (!mediaPlayerIdentifier)
+ return;
+
+ auto* renderer = element.renderer();
+ if (!renderer)
+ return;
+
+ auto rectInRootView = view->contentsToRootView(renderer->videoBox());
+ if (rectInRootView.isEmpty())
+ return;
+
+ send(Messages::WebPageProxy::ExtractVideoInElementFullScreen(mediaPlayerIdentifier, rectInRootView));
+}
+
+void WebPage::cancelVideoExtractionInElementFullScreen()
+{
+ send(Messages::WebPageProxy::CancelVideoExtractionInElementFullScreen());
+}
+
} // namespace WebKit
#undef WEBPAGE_RELEASE_LOG
Modified: trunk/Source/WebKit/WebProcess/WebPage/WebPage.h (292251 => 292252)
--- trunk/Source/WebKit/WebProcess/WebPage/WebPage.h 2022-04-02 02:01:07 UTC (rev 292251)
+++ trunk/Source/WebKit/WebProcess/WebPage/WebPage.h 2022-04-02 02:09:56 UTC (rev 292252)
@@ -200,6 +200,7 @@
class HTMLMenuElement;
class HTMLMenuItemElement;
class HTMLPlugInElement;
+class HTMLVideoElement;
class IntPoint;
class KeyboardEvent;
class MediaPlaybackTargetContext;
@@ -1529,6 +1530,9 @@
bool useSceneKitForModel() const { return m_useSceneKitForModel; };
#endif
+ void extractVideoInElementFullScreen(const WebCore::HTMLVideoElement&);
+ void cancelVideoExtractionInElementFullScreen();
+
private:
WebPage(WebCore::PageIdentifier, WebPageCreationParameters&&);
