Diff
Modified: trunk/ChangeLog (283237 => 283238)
--- trunk/ChangeLog 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/ChangeLog 2021-09-29 18:21:58 UTC (rev 283238)
@@ -1,3 +1,17 @@
+2021-09-29 Kimmo Kinnunen <[email protected]>
+
+ MediaStream canvas.captureStream() fails for WebGL
+ https://bugs.webkit.org/show_bug.cgi?id=230613
+ <rdar://82307293>
+
+ Reviewed by Youenn Fablet.
+
+ Add a manual test to interact with media stream
+ captureStream() behavior until the implementation is
+ testable.
+
+ * ManualTests/mediastream/mediastream-canvas-to-video.html: Added.
+
2021-09-28 Carlos Garcia Campos <[email protected]>
[GTK][WPE] Bump libsoup3 version to 3.0.0
Modified: trunk/LayoutTests/ChangeLog (283237 => 283238)
--- trunk/LayoutTests/ChangeLog 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/LayoutTests/ChangeLog 2021-09-29 18:21:58 UTC (rev 283238)
@@ -1,3 +1,17 @@
+2021-09-29 Kimmo Kinnunen <[email protected]>
+
+ MediaStream canvas.captureStream() fails for WebGL
+ https://bugs.webkit.org/show_bug.cgi?id=230613
+ <rdar://82307293>
+
+ Reviewed by Youenn Fablet.
+
+ Add new expectations, a warning was removed.
+ CONSOLE MESSAGE: Turning drawing buffer preservation for the WebGL canvas being captured
+
+ * fast/mediacapturefromelement/CanvasCaptureMediaStream-webgl-events-expected.txt:
+ * fast/mediastream/captureStream/canvas3d-expected.txt:
+
2021-09-29 Youenn Fablet <[email protected]>
[ iOS macOS wk2 ] webrtc/datachannel/mdns-ice-candidates.html is a flaky failure
Modified: trunk/LayoutTests/fast/mediacapturefromelement/CanvasCaptureMediaStream-webgl-events-expected.txt (283237 => 283238)
--- trunk/LayoutTests/fast/mediacapturefromelement/CanvasCaptureMediaStream-webgl-events-expected.txt 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/LayoutTests/fast/mediacapturefromelement/CanvasCaptureMediaStream-webgl-events-expected.txt 2021-09-29 18:21:58 UTC (rev 283238)
@@ -1,4 +1,3 @@
-CONSOLE MESSAGE: Turning drawing buffer preservation for the WebGL canvas being captured
Exercises the potential events on CanvasCaptureMediaStream.
On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
Modified: trunk/LayoutTests/fast/mediastream/captureStream/canvas3d-expected.txt (283237 => 283238)
--- trunk/LayoutTests/fast/mediastream/captureStream/canvas3d-expected.txt 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/LayoutTests/fast/mediastream/captureStream/canvas3d-expected.txt 2021-09-29 18:21:58 UTC (rev 283238)
@@ -1,4 +1,3 @@
-CONSOLE MESSAGE: Turning drawing buffer preservation for the WebGL canvas being captured
Added: trunk/ManualTests/mediastream/mediastream-canvas-to-video.html (0 => 283238)
--- trunk/ManualTests/mediastream/mediastream-canvas-to-video.html (rev 0)
+++ trunk/ManualTests/mediastream/mediastream-canvas-to-video.html 2021-09-29 18:21:58 UTC (rev 283238)
@@ -0,0 +1,192 @@
+<html>
+<head>
+ <style type="text/css">
+ video { border: solid 1px black; }
+ canvas { padding: 1px }
+ .placeholder {
+ display: inline-block;
+ border: 1px dashed black;
+ width: 300px;
+ height: 150px;
+ }
+ </style>
+</head>
+<body>
+<p>
+<input type="button" _onclick_="drawOnce()" value="Draw once" />
+<input type="button" _onclick_="clearOnce()" value="Clear once" />
+<input type="button" _onclick_="toggleDrawing()" value="Toggle drawing"/>
+</p>
+<p>
+ <ul>
+ <li>Left: canvas</li>
+ <li>Right: video</li>
+ <li>Top rectangle small: opaque</li>
+ <li>Bottom rectangle big: 50% translucent</li>
+ <li>Background: transparent</li>
+ </ul>
+</p>
+<script>
+
+var ctx2Ds = [];
+
+var gls = [];
+
+function createTests()
+{
+ let glCases = [];
+ for (let preserveDrawingBuffer of [true, false]) {
+ for (let alpha of [true, false]) {
+ for (let premultipliedAlpha of [true, false]) {
+ glCases.push({
+ preserveDrawingBuffer: preserveDrawingBuffer,
+ alpha: alpha,
+ premultipliedAlpha: premultipliedAlpha,
+ antialias: false,
+ inDocument: true
+ });
+ }
+ }
+ }
+ glCases.push({
+ preserveDrawingBuffer: false,
+ alpha: true,
+ premultipliedAlpha: true,
+ antialias: true,
+ inDocument: true
+ });
+ glCases.push({
+ preserveDrawingBuffer: false,
+ alpha: true,
+ premultipliedAlpha: true,
+ antialias: false,
+ inDocument: false
+ });
+
+ var ctx2DCases = [];
+ for (let inDocument of [true, false]) {
+ ctx2DCases.push({ inDocument: inDocument });
+ }
+
+ for (let c of glCases) {
+ let d = document.createElement("div");
+ document.body.appendChild(d);
+ let h = document.createElement("h3");
+ h.innerText = `WebGL case: ${JSON.stringify(c)}`;
+ d.appendChild(h);
+ let canvas = document.createElement("canvas");
+ if (c.inDocument)
+ d.appendChild(canvas);
+ else {
+ let p = document.createElement("div");
+ p.className = "placeholder";
+ d.appendChild(p);
+ }
+ let v = document.createElement("video");
+ v.controls = true;
+ v.autoplay = true;
+ d.appendChild(v);
+
+ let gl = canvas.getContext("webgl2", {
+ preserveDrawingBuffer: c.preserveDrawingBuffer,
+ alpha: c.alpha,
+ premultipliedAlpha: c.premultipliedAlpha,
+ antialias: c.antialias
+ });
+ v.srcObject = canvas.captureStream();
+ gls.push(gl);
+ }
+
+ for (let c of ctx2DCases) {
+ let d = document.createElement("div");
+ document.body.appendChild(d);
+ let h = document.createElement("h3");
+ h.innerText = `Canvas case: ${JSON.stringify(c)}`;
+ d.appendChild(h);
+ let canvas = document.createElement("canvas");
+ if (c.inDocument)
+ d.appendChild(canvas);
+ else {
+ let p = document.createElement("div");
+ p.className = "placeholder";
+ d.appendChild(p);
+ }
+ let v = document.createElement("video");
+ v.controls = true;
+ v.autoplay = true;
+ d.appendChild(v);
+ let ctx2D = canvas.getContext("2d");
+ v.srcObject = canvas.captureStream();
+ ctx2Ds.push(ctx2D);
+ }
+}
+
+function drawOnce()
+{
+ for (let c of gls) {
+ c.enable(c.SCISSOR_TEST);
+ if (c.getContextAttributes().premultipliedAlpha)
+ c.clearColor(0., 0.5, 0., 0.5);
+ else
+ c.clearColor(0., 1., 0., 0.5);
+ c.scissor(0, 0, 300, 50);
+ c.clear(c.COLOR_BUFFER_BIT);
+
+ c.scissor(125, 50, 50, 50);
+ c.clearColor(0., 1., 0., 1.);
+ c.clear(c.COLOR_BUFFER_BIT);
+ c.scissor(0, 0, 300, 50);
+ c.clear(c.COLOR_BUFFER_BIT);
+
+ }
+ for (let c of ctx2Ds) {
+ c.fillStyle = "rgba(0, 128, 0, 0.5)";
+ c.fillRect(0, 100, 300, 50);
+ c.fillStyle = "rgba(0, 255, 0, 1.)";
+ c.fillRect(125, 50, 50, 50);
+ }
+}
+
+function clearOnce()
+{
+ console.log("clear");
+ for (let c of gls) {
+ c.clearColor(0., 0., 0., 0.);
+ c.disable(c.SCISSOR_TEST);
+ c.clear(c.COLOR_BUFFER_BIT);
+ }
+ for (let c of ctx2Ds) {
+ c.clearRect(0, 0, 300, 150);
+ }
+}
+
+var isToggling = false;
+
+function toggleDrawing()
+{
+ if (isToggling) {
+ clearTimeout(isToggling);
+ isToggling = false;
+ return;
+ }
+ var hasDrawn = true;
+ function f() {
+ if (hasDrawn)
+ clearOnce();
+ else
+ drawOnce();
+ hasDrawn = !hasDrawn;
+ isToggling = setTimeout(f, 1000);
+ }
+ f();
+}
+
+createTests();
+
+if (window.testRunner) {
+ window.testRunner.waitUntilDone();
+ toggleDrawing();
+}
+</script>
+</body>
+</html>
\ No newline at end of file
Modified: trunk/Source/WebCore/ChangeLog (283237 => 283238)
--- trunk/Source/WebCore/ChangeLog 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/ChangeLog 2021-09-29 18:21:58 UTC (rev 283238)
@@ -1,3 +1,68 @@
+2021-09-29 Kimmo Kinnunen <[email protected]>
+
+ MediaStream canvas.captureStream() fails for WebGL
+ https://bugs.webkit.org/show_bug.cgi?id=230613
+ <rdar://82307293>
+
+ Reviewed by Youenn Fablet.
+
+ Remove previous incorrect behavior of turning on "preservesDrawingBuffer"
+ and sampling the drawing buffer. The drawing buffer is not what should
+ ever be touched, rather it's the display buffer that should be available.
+
+ Implement sampling of WebGL display buffer for
+ media stream capture. The upon sampling, obtain a shared
+ reference to the
+ display buffer, passes it to media sample consuming process and mark
+ the WebGL context display buffer non-eligible for recycling as
+ new drawing buffer.
+
+ Currently the implementation is displayed correctly for
+ WebGL contents that is premultiplied. Unpremultiplied
+ content is fixed in the associated bugs.
+
+ Note: there are bugs preventing robust use of the API,
+ listed in the associated bugs. As such, testing is not expanded
+ in this bug.
+
+ Tested by
+ ManualTests/mediastream/mediastream-canvas-to-video.html (added)
+
+ https://webrtc.github.io/samples/src/content/capture/canvas-video/
+ (Refresh the page or hide the window until MediaPlayer initialization
+ flakyness lets rendering happen)
+
+ The test LayoutTests/fast/mediastream/captureStream/canvas3d.html
+ cannot yet be used to test the implementation as the rest of the
+ captureStream() and playback implementation is so flaky.
+
+ * Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp:
+ (WebCore::CanvasCaptureMediaStreamTrack::Source::canvasChanged):
+ * html/HTMLCanvasElement.cpp:
+ (WebCore::HTMLCanvasElement::toMediaSample):
+ * html/canvas/WebGLRenderingContextBase.cpp:
+ (WebCore::WebGLRenderingContextBase::paintCompositedResultsToMediaSample):
+ * html/canvas/WebGLRenderingContextBase.h:
+ (WebCore::WebGLRenderingContextBase::isPreservingDrawingBuffer const):
+ * platform/graphics/GraphicsContextGL.cpp:
+ * platform/graphics/GraphicsContextGL.h:
+ * platform/graphics/RemoteGraphicsContextGLProxyBase.cpp:
+ (WebCore::RemoteGraphicsContextGLProxyBase::paintCompositedResultsToMediaSample):
+ * platform/graphics/RemoteGraphicsContextGLProxyBase.h:
+ * platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h:
+ * platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm:
+ (WebCore::MediaSampleAVFObjC::createImageSample):
+ * platform/graphics/cocoa/GraphicsContextGLIOSurfaceSwapChain.cpp:
+ (WebCore::GraphicsContextGLIOSurfaceSwapChain::present):
+ * platform/graphics/cocoa/GraphicsContextGLIOSurfaceSwapChain.h:
+ * platform/graphics/cocoa/GraphicsContextGLOpenGLCocoa.mm:
+ (WebCore::GraphicsContextGLOpenGL::paintCompositedResultsToMediaSample):
+ * platform/graphics/cocoa/RemoteGraphicsContextGLProxyBaseCocoa.mm:
+ (WebCore::RemoteGraphicsContextGLProxyBase::paintCompositedResultsToMediaSample):
+ * platform/graphics/opengl/GraphicsContextGLOpenGL.cpp:
+ (WebCore::GraphicsContextGLOpenGL::paintCompositedResultsToMediaSample):
+ * platform/graphics/opengl/GraphicsContextGLOpenGL.h:
+
2021-09-29 Jer Noble <[email protected]>
Build-time optimization: forward declare more things in MediaPlayer.h and HTMLMediaElement.h
Modified: trunk/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp (283237 => 283238)
--- trunk/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp 2021-09-29 18:21:58 UTC (rev 283238)
@@ -153,19 +153,6 @@
{
ASSERT_UNUSED(canvas, m_canvas == &canvas);
-#if ENABLE(WEBGL)
- // FIXME: We need to preserve drawing buffer as we are currently grabbing frames asynchronously.
- // We should instead add an anchor point for both 2d and 3d contexts where canvas will actually paint.
- // And call canvas observers from that point.
- if (is<WebGLRenderingContextBase>(canvas.renderingContext())) {
- auto& context = downcast<WebGLRenderingContextBase>(*canvas.renderingContext());
- if (!context.isPreservingDrawingBuffer()) {
- canvas.scriptExecutionContext()->addConsoleMessage(MessageSource::JS, MessageLevel::Warning, "Turning drawing buffer preservation for the WebGL canvas being captured"_s);
- context.enablePreserveDrawingBuffer();
- }
- }
-#endif
-
// FIXME: We should try to generate the frame at the time the screen is being updated.
if (m_captureCanvasTimer.isActive())
return;
Modified: trunk/Source/WebCore/html/HTMLCanvasElement.cpp (283237 => 283238)
--- trunk/Source/WebCore/html/HTMLCanvasElement.cpp 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/html/HTMLCanvasElement.cpp 2021-09-29 18:21:58 UTC (rev 283238)
@@ -758,6 +758,13 @@
RefPtr<MediaSample> HTMLCanvasElement::toMediaSample()
{
#if PLATFORM(COCOA) || USE(GSTREAMER)
+#if ENABLE(WEBGL)
+ if (is<WebGLRenderingContextBase>(m_context.get())) {
+ if (RuntimeEnabledFeatures::sharedFeatures().webAPIStatisticsEnabled())
+ ResourceLoadObserver::shared().logCanvasRead(document());
+ return downcast<WebGLRenderingContextBase>(*m_context).paintCompositedResultsToMediaSample();
+ }
+#endif
auto* imageBuffer = buffer();
if (!imageBuffer)
return nullptr;
Modified: trunk/Source/WebCore/html/canvas/WebGLRenderingContextBase.cpp (283237 => 283238)
--- trunk/Source/WebCore/html/canvas/WebGLRenderingContextBase.cpp 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/html/canvas/WebGLRenderingContextBase.cpp 2021-09-29 18:21:58 UTC (rev 283238)
@@ -134,6 +134,10 @@
#include "OffscreenCanvas.h"
#endif
+#if ENABLE(MEDIA_STREAM)
+#include "MediaSample.h"
+#endif
+
namespace WebCore {
WTF_MAKE_ISO_ALLOCATED_IMPL(WebGLRenderingContextBase);
@@ -1375,6 +1379,15 @@
return m_context->paintRenderingResultsToPixelBuffer();
}
+#if ENABLE(MEDIA_STREAM)
+RefPtr<MediaSample> WebGLRenderingContextBase::paintCompositedResultsToMediaSample()
+{
+ if (isContextLostOrPending())
+ return nullptr;
+ return m_context->paintCompositedResultsToMediaSample();
+}
+#endif
+
WebGLTexture::TextureExtensionFlag WebGLRenderingContextBase::textureExtensionFlags() const
{
return static_cast<WebGLTexture::TextureExtensionFlag>((m_oesTextureFloatLinear ? WebGLTexture::TextureExtensionFloatLinearEnabled : 0) | (m_oesTextureHalfFloatLinear ? WebGLTexture::TextureExtensionHalfFloatLinearEnabled : 0));
@@ -3950,14 +3963,6 @@
return false;
}
-void WebGLRenderingContextBase::enablePreserveDrawingBuffer()
-{
- ASSERT(!m_attributes.preserveDrawingBuffer);
- m_attributes.preserveDrawingBuffer = true;
- // Must send this notification down to the GraphicsContextGL as well.
- m_context->enablePreserveDrawingBuffer();
-}
-
void WebGLRenderingContextBase::hint(GCGLenum target, GCGLenum mode)
{
if (isContextLostOrPending())
Modified: trunk/Source/WebCore/html/canvas/WebGLRenderingContextBase.h (283237 => 283238)
--- trunk/Source/WebCore/html/canvas/WebGLRenderingContextBase.h 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/html/canvas/WebGLRenderingContextBase.h 2021-09-29 18:21:58 UTC (rev 283238)
@@ -128,6 +128,10 @@
using WebGLCanvas = WTF::Variant<RefPtr<HTMLCanvasElement>>;
#endif
+#if ENABLE(MEDIA_STREAM)
+class MediaSample;
+#endif
+
class WebGLRenderingContextBase : public GraphicsContextGL::Client, public GPUBasedCanvasRenderingContext, private ActivityStateChangeObserver {
WTF_MAKE_ISO_ALLOCATED(WebGLRenderingContextBase);
public:
@@ -233,10 +237,6 @@
bool extensionIsEnabled(const String&);
bool isPreservingDrawingBuffer() const { return m_attributes.preserveDrawingBuffer; }
- // Concession to canvas capture API, which must dynamically enable
- // preserveDrawingBuffer. This can only be called once, when
- // isPreservingDrawingBuffer() returns false.
- void enablePreserveDrawingBuffer();
bool preventBufferClearForInspector() const { return m_preventBufferClearForInspector; }
void setPreventBufferClearForInspector(bool value) { m_preventBufferClearForInspector = value; }
@@ -392,6 +392,9 @@
void prepareForDisplayWithPaint() final;
void paintRenderingResultsToCanvas() final;
std::optional<PixelBuffer> paintRenderingResultsToPixelBuffer();
+#if ENABLE(MEDIA_STREAM)
+ RefPtr<MediaSample> paintCompositedResultsToMediaSample();
+#endif
void removeSharedObject(WebGLSharedObject&);
void removeContextObject(WebGLContextObject&);
Modified: trunk/Source/WebCore/platform/graphics/GraphicsContextGL.cpp (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/GraphicsContextGL.cpp 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/GraphicsContextGL.cpp 2021-09-29 18:21:58 UTC (rev 283238)
@@ -362,12 +362,6 @@
{
}
-void GraphicsContextGL::enablePreserveDrawingBuffer()
-{
- // Canvas capture should not call this unless necessary.
- ASSERT(!m_attrs.preserveDrawingBuffer);
- m_attrs.preserveDrawingBuffer = true;
-}
bool GraphicsContextGL::computeFormatAndTypeParameters(GCGLenum format, GCGLenum type, unsigned* componentsPerPixel, unsigned* bytesPerComponent)
{
Modified: trunk/Source/WebCore/platform/graphics/GraphicsContextGL.h (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/GraphicsContextGL.h 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/GraphicsContextGL.h 2021-09-29 18:21:58 UTC (rev 283238)
@@ -59,8 +59,10 @@
#if ENABLE(VIDEO) && USE(AVFOUNDATION)
class GraphicsContextGLCV;
#endif
+#if ENABLE(MEDIA_STREAM)
+class MediaSample;
+#endif
-
// Base class for graphics context for implementing WebGL rendering model.
class GraphicsContextGL : public RefCounted<GraphicsContextGL> {
public:
@@ -1244,13 +1246,6 @@
GraphicsContextGLAttributes contextAttributes() const { return m_attrs; }
void setContextAttributes(const GraphicsContextGLAttributes& attrs) { m_attrs = attrs; }
- // FIXME: The calling site should be fixed, this breaks the WebGL spec
- // wrt preserveDrawingBuffer == false behavior.
- // Concession to Canvas captureStream, which needs to dynamically set
- // preserveDrawingBuffer to true in order to avoid implicit clears.
- // Implementations generally do not support toggling this bit arbitrarily.
- virtual void enablePreserveDrawingBuffer();
-
// Support for extensions. Returns a non-null object, though not
// all methods it contains may necessarily be supported on the
// current hardware. Must call ExtensionsGL::supports() to
@@ -1294,6 +1289,9 @@
virtual void paintRenderingResultsToCanvas(ImageBuffer&) = 0;
virtual std::optional<PixelBuffer> paintRenderingResultsToPixelBuffer() = 0;
virtual void paintCompositedResultsToCanvas(ImageBuffer&) = 0;
+#if ENABLE(MEDIA_STREAM)
+ virtual RefPtr<MediaSample> paintCompositedResultsToMediaSample() = 0;
+#endif
// FIXME: this should be removed. The layer should be marked composited by
// preparing for display, so that canvas image buffer and the layer agree
Modified: trunk/Source/WebCore/platform/graphics/RemoteGraphicsContextGLProxyBase.cpp (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/RemoteGraphicsContextGLProxyBase.cpp 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/RemoteGraphicsContextGLProxyBase.cpp 2021-09-29 18:21:58 UTC (rev 283238)
@@ -97,13 +97,6 @@
return m_buffersToAutoClear;
}
-void RemoteGraphicsContextGLProxyBase::enablePreserveDrawingBuffer()
-{
- // Redeclared for export reasons.
- // FIXME: The whole function should be removed.
- GraphicsContextGL::enablePreserveDrawingBuffer();
-}
-
bool RemoteGraphicsContextGLProxyBase::supports(const String& name)
{
waitUntilInitialized();
@@ -156,5 +149,12 @@
{
}
#endif
+
+#if ENABLE(MEDIA_STREAM) && !PLATFORM(COCOA)
+RefPtr<MediaSample> RemoteGraphicsContextGLProxyBase::paintCompositedResultsToMediaSample()
+{
+ return nullptr;
}
#endif
+}
+#endif
Modified: trunk/Source/WebCore/platform/graphics/RemoteGraphicsContextGLProxyBase.h (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/RemoteGraphicsContextGLProxyBase.h 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/RemoteGraphicsContextGLProxyBase.h 2021-09-29 18:21:58 UTC (rev 283238)
@@ -67,7 +67,6 @@
void setBuffersToAutoClear(GCGLbitfield) final;
GCGLbitfield getBuffersToAutoClear() const final;
void markLayerComposited() final;
- void enablePreserveDrawingBuffer() final;
#if ENABLE(VIDEO) && USE(AVFOUNDATION)
GraphicsContextGLCV* asCV() final;
@@ -83,6 +82,9 @@
void getnUniformfvEXT(GCGLuint, GCGLint, GCGLsizei, GCGLfloat*) final;
void getnUniformivEXT(GCGLuint, GCGLint, GCGLsizei, GCGLint*) final;
#endif
+#if ENABLE(MEDIA_STREAM)
+ RefPtr<MediaSample> paintCompositedResultsToMediaSample() final;
+#endif
protected:
void initialize(const String& availableExtensions, const String& requestableExtensions);
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h 2021-09-29 18:21:58 UTC (rev 283238)
@@ -30,6 +30,8 @@
#include <pal/avfoundation/MediaTimeAVFoundation.h>
#include <wtf/Forward.h>
+using CVPixelBufferRef = struct __CVBuffer*;
+
namespace WebCore {
class PixelBuffer;
@@ -40,6 +42,7 @@
static Ref<MediaSampleAVFObjC> create(CMSampleBufferRef sample, AtomString trackID) { return adoptRef(*new MediaSampleAVFObjC(sample, trackID)); }
static Ref<MediaSampleAVFObjC> create(CMSampleBufferRef sample, VideoRotation rotation = VideoRotation::None, bool mirrored = false) { return adoptRef(*new MediaSampleAVFObjC(sample, rotation, mirrored)); }
static RefPtr<MediaSampleAVFObjC> createImageSample(PixelBuffer&&);
+ static RefPtr<MediaSampleAVFObjC> createImageSample(RetainPtr<CVPixelBufferRef>&&, VideoRotation, bool mirrored);
WEBCORE_EXPORT static void setAsDisplayImmediately(MediaSample&);
static RetainPtr<CMSampleBufferRef> cloneSampleBufferAndSetAsDisplayImmediately(CMSampleBufferRef);
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm 2021-09-29 18:21:58 UTC (rev 283238)
@@ -95,24 +95,33 @@
return nullptr;
}
ASSERT_UNUSED(status, !status);
+ return createImageSample(WTFMove(cvPixelBuffer), VideoRotation::None, false);
+}
+RefPtr<MediaSampleAVFObjC> MediaSampleAVFObjC::createImageSample(RetainPtr<CVPixelBufferRef>&& pixelBuffer, VideoRotation rotation, bool mirrored)
+{
CMVideoFormatDescriptionRef formatDescriptionRaw = nullptr;
- status = PAL::CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, cvPixelBuffer.get(), &formatDescriptionRaw);
- ASSERT(!status);
+ auto status = PAL::CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer.get(), &formatDescriptionRaw);
+ if (status || !formatDescriptionRaw) {
+ ASSERT_NOT_REACHED();
+ return nullptr;
+ }
auto formatDescription = adoptCF(formatDescriptionRaw);
CMSampleTimingInfo sampleTimingInformation = { PAL::kCMTimeInvalid, PAL::kCMTimeInvalid, PAL::kCMTimeInvalid };
CMSampleBufferRef sampleBufferRaw = nullptr;
- status = PAL::CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, cvPixelBuffer.get(), formatDescription.get(), &sampleTimingInformation, &sampleBufferRaw);
- ASSERT(!status);
+ status = PAL::CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBuffer.get(), formatDescription.get(), &sampleTimingInformation, &sampleBufferRaw);
+ if (status || !sampleBufferRaw) {
+ ASSERT_NOT_REACHED();
+ return nullptr;
+ }
auto sampleBuffer = adoptCF(sampleBufferRaw);
-
CFArrayRef attachmentsArray = PAL::CMSampleBufferGetSampleAttachmentsArray(sampleBuffer.get(), true);
for (CFIndex i = 0, count = CFArrayGetCount(attachmentsArray); i < count; ++i) {
CFMutableDictionaryRef attachments = checked_cf_cast<CFMutableDictionaryRef>(CFArrayGetValueAtIndex(attachmentsArray, i));
CFDictionarySetValue(attachments, PAL::kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
}
- return create(sampleBuffer.get());
+ return create(sampleBuffer.get(), rotation, mirrored);
}
MediaTime MediaSampleAVFObjC::presentationTime() const
Modified: trunk/Source/WebCore/platform/graphics/cocoa/GraphicsContextGLIOSurfaceSwapChain.cpp (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/cocoa/GraphicsContextGLIOSurfaceSwapChain.cpp 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/cocoa/GraphicsContextGLIOSurfaceSwapChain.cpp 2021-09-29 18:21:58 UTC (rev 283238)
@@ -37,6 +37,11 @@
return m_displayBuffer;
}
+void GraphicsContextGLIOSurfaceSwapChain::markDisplayBufferInUse()
+{
+ m_displayBufferInUse = true;
+}
+
GraphicsContextGLIOSurfaceSwapChain::Buffer GraphicsContextGLIOSurfaceSwapChain::recycleBuffer()
{
if (m_spareBuffer.surface) {
@@ -56,6 +61,10 @@
void GraphicsContextGLIOSurfaceSwapChain::present(Buffer&& buffer)
{
m_spareBuffer = std::exchange(m_displayBuffer, WTFMove(buffer));
+ if (m_displayBufferInUse) {
+ m_displayBufferInUse = false;
+ m_spareBuffer.surface.reset();
+ }
}
}
Modified: trunk/Source/WebCore/platform/graphics/cocoa/GraphicsContextGLIOSurfaceSwapChain.h (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/cocoa/GraphicsContextGLIOSurfaceSwapChain.h 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/cocoa/GraphicsContextGLIOSurfaceSwapChain.h 2021-09-29 18:21:58 UTC (rev 283238)
@@ -67,6 +67,9 @@
// next present, recycleBuffer or detachClient call.
virtual const Buffer& displayBuffer() const;
+ // Marks the current display buffer to be in use and not eligble to recycling.
+ void markDisplayBufferInUse();
+
// Detaches the client and returns the current contents buffer metadata handle.
// The if multiple buffers have been submitted, recycleBuffer must have been called before calling
// this.
@@ -75,6 +78,7 @@
protected:
Buffer m_displayBuffer;
Buffer m_spareBuffer;
+ bool m_displayBufferInUse = false;
};
}
Modified: trunk/Source/WebCore/platform/graphics/cocoa/GraphicsContextGLOpenGLCocoa.mm (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/cocoa/GraphicsContextGLOpenGLCocoa.mm 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/cocoa/GraphicsContextGLOpenGLCocoa.mm 2021-09-29 18:21:58 UTC (rev 283238)
@@ -28,6 +28,7 @@
#if ENABLE(WEBGL)
#import "GraphicsContextGLOpenGL.h"
+#import "CVUtilities.h"
#import "ExtensionsGLANGLE.h"
#import "GraphicsContextGLANGLEUtilities.h"
#import "GraphicsContextGLIOSurfaceSwapChain.h"
@@ -50,6 +51,10 @@
#include "GraphicsContextGLCV.h"
#endif
+#if ENABLE(MEDIA_STREAM)
+#include "MediaSampleAVFObjC.h"
+#endif
+
WTF_WEAK_LINK_FORCE_IMPORT(EGL_Initialize);
namespace WebCore {
@@ -878,6 +883,21 @@
ScopedEGLDefaultDisplay::releaseAllResourcesIfUnused();
}
+#if ENABLE(MEDIA_STREAM)
+RefPtr<MediaSample> GraphicsContextGLOpenGL::paintCompositedResultsToMediaSample()
+{
+ auto &displayBuffer = m_swapChain->displayBuffer();
+ if (!displayBuffer.surface || !displayBuffer.handle)
+ return nullptr;
+ if (displayBuffer.surface->size() != getInternalFramebufferSize())
+ return nullptr;
+ m_swapChain->markDisplayBufferInUse();
+ auto pixelBuffer = createCVPixelBuffer(displayBuffer.surface->surface());
+ if (!pixelBuffer)
+ return nullptr;
+ return MediaSampleAVFObjC::createImageSample(WTFMove(*pixelBuffer), MediaSampleAVFObjC::VideoRotation::UpsideDown, true);
}
+#endif
+}
#endif // ENABLE(WEBGL)
Modified: trunk/Source/WebCore/platform/graphics/cocoa/RemoteGraphicsContextGLProxyBaseCocoa.mm (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/cocoa/RemoteGraphicsContextGLProxyBaseCocoa.mm 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/cocoa/RemoteGraphicsContextGLProxyBaseCocoa.mm 2021-09-29 18:21:58 UTC (rev 283238)
@@ -32,6 +32,11 @@
#import "WebGLLayer.h"
#import <wtf/BlockObjCExceptions.h>
+#if ENABLE(MEDIA_STREAM)
+#import "CVUtilities.h"
+#import "MediaSampleAVFObjC.h"
+#endif
+
namespace WebCore {
void RemoteGraphicsContextGLProxyBase::platformInitialize()
@@ -63,5 +68,20 @@
return [m_webGLLayer swapChain];
}
+#if ENABLE(MEDIA_STREAM)
+RefPtr<MediaSample> RemoteGraphicsContextGLProxyBase::paintCompositedResultsToMediaSample()
+{
+ auto& sc = platformSwapChain();
+ auto& displayBuffer = sc.displayBuffer();
+ if (!displayBuffer.surface)
+ return nullptr;
+ sc.markDisplayBufferInUse();
+ auto pixelBuffer = createCVPixelBuffer(displayBuffer.surface->surface());
+ if (!pixelBuffer)
+ return nullptr;
+ return MediaSampleAVFObjC::createImageSample(WTFMove(*pixelBuffer), MediaSampleAVFObjC::VideoRotation::UpsideDown, true);
}
#endif
+
+}
+#endif
Modified: trunk/Source/WebCore/platform/graphics/opengl/GraphicsContextGLOpenGL.cpp (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/opengl/GraphicsContextGLOpenGL.cpp 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/opengl/GraphicsContextGLOpenGL.cpp 2021-09-29 18:21:58 UTC (rev 283238)
@@ -34,6 +34,7 @@
#include "ImageBuffer.h"
#include "MediaPlayerPrivate.h"
#include "PixelBuffer.h"
+#include <memory>
#include <wtf/UniqueArray.h>
#if USE(AVFOUNDATION)
@@ -40,7 +41,9 @@
#include "GraphicsContextGLCV.h"
#endif
-#include <memory>
+#if !PLATFORM(COCOA) && ENABLE(MEDIA_STREAM)
+#include "MediaSample.h"
+#endif
namespace WebCore {
@@ -72,14 +75,6 @@
return m_buffersToAutoClear;
}
-void GraphicsContextGLOpenGL::enablePreserveDrawingBuffer()
-{
- GraphicsContextGL::enablePreserveDrawingBuffer();
- // After dynamically transitioning to preserveDrawingBuffer:true
- // for canvas capture, clear out any buffer auto-clearing state.
- m_buffersToAutoClear = 0;
-}
-
#if !USE(ANGLE)
bool GraphicsContextGLOpenGL::texImage2DResourceSafe(GCGLenum target, GCGLint level, GCGLenum internalformat, GCGLsizei width, GCGLsizei height, GCGLint border, GCGLenum format, GCGLenum type, GCGLint unpackAlignment)
{
@@ -170,6 +165,13 @@
return results;
}
+#if !PLATFORM(COCOA) && ENABLE(MEDIA_STREAM)
+RefPtr<MediaSample> GraphicsContextGLOpenGL::paintCompositedResultsToMediaSample()
+{
+ return nullptr;
+}
+#endif
+
std::optional<PixelBuffer> GraphicsContextGLOpenGL::readRenderingResultsForPainting()
{
if (!makeContextCurrent())
Modified: trunk/Source/WebCore/platform/graphics/opengl/GraphicsContextGLOpenGL.h (283237 => 283238)
--- trunk/Source/WebCore/platform/graphics/opengl/GraphicsContextGLOpenGL.h 2021-09-29 18:19:46 UTC (rev 283237)
+++ trunk/Source/WebCore/platform/graphics/opengl/GraphicsContextGLOpenGL.h 2021-09-29 18:21:58 UTC (rev 283238)
@@ -460,7 +460,6 @@
void resetBuffersToAutoClear();
void setBuffersToAutoClear(GCGLbitfield) final;
GCGLbitfield getBuffersToAutoClear() const final;
- void enablePreserveDrawingBuffer() final;
void dispatchContextChangedNotification();
@@ -467,6 +466,9 @@
void paintRenderingResultsToCanvas(ImageBuffer&) final;
std::optional<PixelBuffer> paintRenderingResultsToPixelBuffer() final;
void paintCompositedResultsToCanvas(ImageBuffer&) final;
+#if ENABLE(MEDIA_STREAM)
+ RefPtr<MediaSample> paintCompositedResultsToMediaSample() final;
+#endif
std::optional<PixelBuffer> readRenderingResultsForPainting();
std::optional<PixelBuffer> readCompositedResultsForPainting();
