Diff
Modified: trunk/LayoutTests/ChangeLog (280719 => 280720)
--- trunk/LayoutTests/ChangeLog 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/LayoutTests/ChangeLog 2021-08-06 09:15:49 UTC (rev 280720)
@@ -1,3 +1,14 @@
+2021-08-06 Youenn Fablet <[email protected]>
+
+ MediaPlayerPrivateMediaStreamAVFObjC should skip enqueuing frames when not visible
+ https://bugs.webkit.org/show_bug.cgi?id=228816
+ <rdar://81077972>
+
+ Reviewed by Eric Carlson.
+
+ * fast/mediastream/mediaPlayer-visibility-expected.txt: Added.
+ * fast/mediastream/mediaPlayer-visibility.html: Added.
+
2021-08-05 Ryan Haddad <[email protected]>
Unreviewed test garening, skip test that is unsupported on WK1.
Added: trunk/LayoutTests/fast/mediastream/mediaPlayer-visibility-expected.txt (0 => 280720)
--- trunk/LayoutTests/fast/mediastream/mediaPlayer-visibility-expected.txt (rev 0)
+++ trunk/LayoutTests/fast/mediastream/mediaPlayer-visibility-expected.txt 2021-08-06 09:15:49 UTC (rev 280720)
@@ -0,0 +1,6 @@
+
+
+PASS Check default media stream player visibility
+PASS Check media stream player visibility with CSS
+PASS Check media stream player visibility of unattached media element
+
Added: trunk/LayoutTests/fast/mediastream/mediaPlayer-visibility.html (0 => 280720)
--- trunk/LayoutTests/fast/mediastream/mediaPlayer-visibility.html (rev 0)
+++ trunk/LayoutTests/fast/mediastream/mediaPlayer-visibility.html 2021-08-06 09:15:49 UTC (rev 280720)
@@ -0,0 +1,59 @@
+<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
+<html>
+<body>
+<video id="video1" autoplay playsInline></video>
+<video id="video2" autoplay playsInline style="visibility:hidden"></video>
+<script src=""
+<script src=""
+<script>
+
+promise_test(async() => {
+ if (!window.internals)
+ return Promise.reject("Test requires internals API");
+
+ const stream = await navigator.mediaDevices.getUserMedia({video : true});
+ video1.srcObject = stream;
+ await video1.play();
+
+ assert_true(internals.isPlayerVisibleInViewport(video1), "video1");
+}, "Check default media stream player visibility");
+
+promise_test(async() => {
+ if (!window.internals)
+ return Promise.reject("Test requires internals API");
+
+ const stream = await navigator.mediaDevices.getUserMedia({video : true});
+ video2.srcObject = stream;
+ await video2.play();
+
+ assert_false(internals.isPlayerVisibleInViewport(video2), "video2");
+
+ video2.style.visibility = "visible";
+ await new Promise(resolve => setTimeout(resolve, 0));
+ assert_true(internals.isPlayerVisibleInViewport(video2), "video2 2");
+
+ video2.style.visibility = "hidden";
+ await new Promise(resolve => setTimeout(resolve, 0));
+ assert_false(internals.isPlayerVisibleInViewport(video2), "video2 3");
+}, "Check media stream player visibility with CSS");
+
+promise_test(async() => {
+ if (!window.internals)
+ return Promise.reject("Test requires internals API");
+
+ const video3 = document.createElement("video");
+ const stream = await navigator.mediaDevices.getUserMedia({video : true});
+ video3.srcObject = stream;
+ video3.playsInline = true;
+ await video3.play();
+
+ assert_false(internals.isPlayerVisibleInViewport(video3), "video3");
+
+ document.body.appendChild(video3);
+ await video3.play();
+ await new Promise(resolve => setTimeout(resolve, 0));
+ assert_false(internals.isPlayerVisibleInViewport(video3), "video3 2");
+}, "Check media stream player visibility of unattached media element");
+</script>
+</body>
+</html>
Modified: trunk/Source/WebCore/ChangeLog (280719 => 280720)
--- trunk/Source/WebCore/ChangeLog 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/Source/WebCore/ChangeLog 2021-08-06 09:15:49 UTC (rev 280720)
@@ -1,3 +1,45 @@
+2021-08-06 Youenn Fablet <[email protected]>
+
+ MediaPlayerPrivateMediaStreamAVFObjC should skip enqueuing frames when not visible
+ https://bugs.webkit.org/show_bug.cgi?id=228816
+ <rdar://81077972>
+
+ Reviewed by Eric Carlson.
+
+ Do not create layers when not needed.
+ This prevents enqueuing frames in a AVSampleBufferDisplayLayer that will only buffer them, thus breaking camera capture/video decoding.
+ We do this by not calling ensureLayers when getting a new track. Instead we react upon player renderingCanBeAccelerated value.
+
+ It is also unnecessary and takes CPU cycles to enqueue frames when the video element is not visible.
+ HTMLMediaElement and RenderVideo thus pass to MediaPlayer a new flag telling whether the video element is visible in the view port.
+ MediaPlayerPrivateMediaStreamAVFObjC will then skip enqueueing frames if not needed.
+
+ Add getter and internals API to cover these changes.
+
+ Test: fast/mediastream/mediaPlayer-visibility.html
+
+ * html/HTMLMediaElement.cpp:
+ (WebCore::HTMLMediaElement::isVisibleInViewportChanged):
+ * platform/graphics/MediaPlayer.cpp:
+ (WebCore::MediaPlayer::loadWithNextMediaEngine):
+ (WebCore::MediaPlayer::setVisibleInViewport):
+ * platform/graphics/MediaPlayer.h:
+ * platform/graphics/MediaPlayerPrivate.h:
+ (WebCore::MediaPlayerPrivateInterface::setVisibleInViewport):
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSample):
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::setVisible):
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::setVisibleInViewport):
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::checkSelectedVideoTrack):
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext):
+ * rendering/RenderVideo.cpp:
+ (WebCore::RenderVideo::updatePlayer):
+ * testing/Internals.cpp:
+ (WebCore::Internals::isPlayerVisibleInViewport const):
+ * testing/Internals.h:
+ * testing/Internals.idl:
+
2021-08-05 Cameron McCormack <[email protected]>
Stop tracking form elements with FormController
Modified: trunk/Source/WebCore/html/HTMLMediaElement.cpp (280719 => 280720)
--- trunk/Source/WebCore/html/HTMLMediaElement.cpp 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/Source/WebCore/html/HTMLMediaElement.cpp 2021-08-06 09:15:49 UTC (rev 280720)
@@ -6744,6 +6744,7 @@
m_player->setPreferredDynamicRangeMode(m_overrideDynamicRangeMode.value_or(preferredDynamicRangeMode(document().view())));
m_player->setMuted(effectiveMuted());
m_player->setVisible(!m_elementIsHidden);
+ m_player->setVisibleInViewport(isVisibleInViewport());
schedulePlaybackControlsManagerUpdate();
#if ENABLE(WEB_AUDIO)
@@ -8071,6 +8072,8 @@
void HTMLMediaElement::isVisibleInViewportChanged()
{
+ if (m_player)
+ m_player->setVisibleInViewport(isVisibleInViewport());
queueTaskKeepingObjectAlive(*this, TaskSource::MediaElement, [this] {
if (isContextStopped())
return;
Modified: trunk/Source/WebCore/platform/graphics/MediaPlayer.cpp (280719 => 280720)
--- trunk/Source/WebCore/platform/graphics/MediaPlayer.cpp 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/Source/WebCore/platform/graphics/MediaPlayer.cpp 2021-08-06 09:15:49 UTC (rev 280720)
@@ -575,6 +575,8 @@
client().mediaPlayerEngineUpdated();
if (m_visible)
m_private->setVisible(m_visible);
+ if (m_visibleInViewport)
+ m_private->setVisibleInViewport(m_visibleInViewport);
m_private->prepareForPlayback(m_privateBrowsing, m_preload, m_preservesPitch, m_shouldPrepareToRender);
}
}
@@ -1011,6 +1013,12 @@
m_private->setVisibleForCanvas(visible);
}
+void MediaPlayer::setVisibleInViewport(bool visible)
+{
+ m_visibleInViewport = visible;
+ m_private->setVisibleInViewport(visible);
+}
+
MediaPlayer::Preload MediaPlayer::preload() const
{
return m_preload;
Modified: trunk/Source/WebCore/platform/graphics/MediaPlayer.h (280719 => 280720)
--- trunk/Source/WebCore/platform/graphics/MediaPlayer.h 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/Source/WebCore/platform/graphics/MediaPlayer.h 2021-08-06 09:15:49 UTC (rev 280720)
@@ -360,6 +360,9 @@
void setVisible(bool);
void setVisibleForCanvas(bool);
+ void setVisibleInViewport(bool);
+ bool isVisibleInViewport() const { return m_visibleInViewport; }
+
void prepareToPlay();
void play();
void pause();
@@ -699,6 +702,7 @@
Preload m_preload { Preload::Auto };
double m_volume { 1 };
bool m_visible { false };
+ bool m_visibleInViewport { false };
bool m_muted { false };
bool m_preservesPitch { true };
bool m_privateBrowsing { false };
Modified: trunk/Source/WebCore/platform/graphics/MediaPlayerPrivate.h (280719 => 280720)
--- trunk/Source/WebCore/platform/graphics/MediaPlayerPrivate.h 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/Source/WebCore/platform/graphics/MediaPlayerPrivate.h 2021-08-06 09:15:49 UTC (rev 280720)
@@ -98,6 +98,7 @@
virtual void setVisible(bool) = 0;
virtual void setVisibleForCanvas(bool visible) { setVisible(visible); }
+ virtual void setVisibleInViewport(bool) { }
virtual float duration() const { return 0; }
virtual double durationDouble() const { return duration(); }
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h (280719 => 280720)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h 2021-08-06 09:15:49 UTC (rev 280720)
@@ -122,6 +122,7 @@
void setVisible(bool) final;
void setVisibleForCanvas(bool) final;
+ void setVisibleInViewport(bool) final;
MediaTime durationMediaTime() const override;
MediaTime currentMediaTime() const override;
@@ -143,7 +144,7 @@
void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
bool metaDataAvailable() const { return m_mediaStreamPrivate && m_readyState >= MediaPlayer::ReadyState::HaveMetadata; }
- void acceleratedRenderingStateChanged() override;
+ void acceleratedRenderingStateChanged() final { updateLayersAsNeeded(); }
bool supportsAcceleratedRendering() const override { return true; }
bool hasSingleSecurityOrigin() const override { return true; }
@@ -170,6 +171,7 @@
void scheduleDeferredTask(Function<void ()>&&);
void layersAreInitialized(IntSize, bool);
+ void updateLayersAsNeeded();
enum DisplayMode {
None,
@@ -272,7 +274,8 @@
bool m_muted { false };
bool m_ended { false };
bool m_hasEverEnqueuedVideoFrame { false };
- bool m_visible { false };
+ bool m_isPageVisible { false };
+ bool m_isVisibleInViewPort { false };
bool m_haveSeenMetadata { false };
bool m_waitingForFirstImage { false };
};
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm (280719 => 280720)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm 2021-08-06 09:15:49 UTC (rev 280720)
@@ -252,7 +252,7 @@
void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSample(MediaSample& sample)
{
- if (!m_visible)
+ if (!m_isPageVisible || !m_isVisibleInViewPort)
return;
if (!m_sampleBufferDisplayLayerLock.tryLock())
@@ -614,12 +614,12 @@
return !m_audioTrackMap.isEmpty();
}
-void MediaPlayerPrivateMediaStreamAVFObjC::setVisible(bool visible)
+void MediaPlayerPrivateMediaStreamAVFObjC::setVisible(bool isVisible)
{
- if (m_visible == visible)
+ if (m_isPageVisible == isVisible)
return;
- m_visible = visible;
+ m_isPageVisible = isVisible;
flushRenderers();
}
@@ -627,6 +627,11 @@
{
}
+void MediaPlayerPrivateMediaStreamAVFObjC::setVisibleInViewport(bool isVisible)
+{
+ m_isVisibleInViewPort = isVisible;
+}
+
MediaTime MediaPlayerPrivateMediaStreamAVFObjC::durationMediaTime() const
{
return MediaTime::positiveInfiniteTime();
@@ -861,7 +866,9 @@
if (m_displayMode == None)
m_waitingForFirstImage = true;
}
- ensureLayers();
+
+ updateLayersAsNeeded();
+
if (m_sampleBufferDisplayLayer) {
if (!m_activeVideoTrack)
m_sampleBufferDisplayLayer->clearEnqueuedSamples();
@@ -998,7 +1005,7 @@
context.drawNativeImage(*image, imageRect.size(), transformedDestRect, imageRect);
}
-void MediaPlayerPrivateMediaStreamAVFObjC::acceleratedRenderingStateChanged()
+void MediaPlayerPrivateMediaStreamAVFObjC::updateLayersAsNeeded()
{
if (m_player->renderingCanBeAccelerated())
ensureLayers();
Modified: trunk/Source/WebCore/rendering/RenderVideo.cpp (280719 => 280720)
--- trunk/Source/WebCore/rendering/RenderVideo.cpp 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/Source/WebCore/rendering/RenderVideo.cpp 2021-08-06 09:15:49 UTC (rev 280720)
@@ -282,6 +282,7 @@
IntRect videoBounds = videoBox();
mediaPlayer->setSize(IntSize(videoBounds.width(), videoBounds.height()));
mediaPlayer->setVisible(!videoElement().elementIsHidden());
+ mediaPlayer->setVisibleInViewport(videoElement().isVisibleInViewport());
mediaPlayer->setShouldMaintainAspectRatio(style().objectFit() != ObjectFit::Fill);
}
Modified: trunk/Source/WebCore/testing/Internals.cpp (280719 => 280720)
--- trunk/Source/WebCore/testing/Internals.cpp 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/Source/WebCore/testing/Internals.cpp 2021-08-06 09:15:49 UTC (rev 280720)
@@ -4420,6 +4420,13 @@
{
return element.isDisablingSleep();
}
+
+bool Internals::isPlayerVisibleInViewport(HTMLMediaElement& element) const
+{
+ auto player = element.player();
+ return player && player->isVisibleInViewport();
+}
+
#endif // ENABLE(VIDEO)
#if ENABLE(WEB_AUDIO)
Modified: trunk/Source/WebCore/testing/Internals.h (280719 => 280720)
--- trunk/Source/WebCore/testing/Internals.h 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/Source/WebCore/testing/Internals.h 2021-08-06 09:15:49 UTC (rev 280720)
@@ -713,6 +713,7 @@
ExceptionOr<void> postRemoteControlCommand(const String&, float argument);
void activeAudioRouteDidChange(bool shouldPause);
bool elementIsBlockingDisplaySleep(HTMLMediaElement&) const;
+ bool isPlayerVisibleInViewport(HTMLMediaElement&) const;
#endif
#if ENABLE(WIRELESS_PLAYBACK_TARGET)
Modified: trunk/Source/WebCore/testing/Internals.idl (280719 => 280720)
--- trunk/Source/WebCore/testing/Internals.idl 2021-08-06 03:30:45 UTC (rev 280719)
+++ trunk/Source/WebCore/testing/Internals.idl 2021-08-06 09:15:49 UTC (rev 280720)
@@ -773,6 +773,7 @@
[Conditional=VIDEO] undefined simulateSystemSleep();
[Conditional=VIDEO] undefined simulateSystemWake();
[Conditional=VIDEO] boolean elementIsBlockingDisplaySleep(HTMLMediaElement element);
+ [Conditional=VIDEO] boolean isPlayerVisibleInViewport(HTMLMediaElement element);
MockPageOverlay installMockPageOverlay(PageOverlayType type);
DOMString pageOverlayLayerTreeAsText(optional unsigned short flags = 0);
