Diff
Modified: trunk/Source/WebKit/ChangeLog (290823 => 290824)
--- trunk/Source/WebKit/ChangeLog 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/ChangeLog 2022-03-04 09:15:08 UTC (rev 290824)
@@ -1,3 +1,34 @@
+2022-03-04 Youenn Fablet <[email protected]>
+
+ Introduce routines to exchange SharedVideoFrame::Buffer directly
+ https://bugs.webkit.org/show_bug.cgi?id=237267
+
+ Reviewed by Eric Carlson.
+
+ Define explictly SharedVideoFrame::Buffer as Variant of the various buffer representations.
+ This allows to move more code in SharedVideoFrame for instance by adding direct support of sending webrtc::VideoFrame buffers in SharedVideoFrame.
+ Reuse that code in LibWebRTCCodecs as a refactoring/simplification.
+ Reuse that code in RemoteVideoFrameObjectHeap which allows to exchange IOSurfaces in case WebProcess can use them.
+ This is used when encoding such frames using software encoders.
+
+ Covered by existing tests.
+
+ * GPUProcess/media/RemoteVideoFrameObjectHeap.cpp:
+ * GPUProcess/media/RemoteVideoFrameObjectHeap.h:
+ * GPUProcess/media/RemoteVideoFrameObjectHeap.messages.in:
+ * GPUProcess/webrtc/LibWebRTCCodecsProxy.h:
+ * GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in:
+ * GPUProcess/webrtc/LibWebRTCCodecsProxy.mm:
+ * WebKit.xcodeproj/project.pbxproj:
+ * WebProcess/GPU/media/RemoteVideoFrameProxy.cpp:
+ * WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp:
+ * WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxy.h:
+ * WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.cpp:
+ * WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.h:
+ * WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.messages.in:
+ * WebProcess/GPU/webrtc/SharedVideoFrame.cpp:
+ * WebProcess/GPU/webrtc/SharedVideoFrame.h:
+
2022-03-03 Brady Eidson <[email protected]>
Add a new "Daemon -> Client" message type to webpushd
Modified: trunk/Source/WebKit/GPUProcess/media/RemoteVideoFrameObjectHeap.cpp (290823 => 290824)
--- trunk/Source/WebKit/GPUProcess/media/RemoteVideoFrameObjectHeap.cpp 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteVideoFrameObjectHeap.cpp 2022-03-04 09:15:08 UTC (rev 290824)
@@ -101,35 +101,22 @@
}
#if PLATFORM(COCOA)
-void RemoteVideoFrameObjectHeap::getVideoFrameBuffer(RemoteVideoFrameReadReference&& read)
+void RemoteVideoFrameObjectHeap::getVideoFrameBuffer(RemoteVideoFrameReadReference&& read, bool canSendIOSurface)
{
assertIsCurrent(remoteVideoFrameObjectHeapQueue());
auto identifier = read.identifier();
- auto videoFrame = m_heap.retire(WTFMove(read), 0_s);
+ auto videoFrame = get(WTFMove(read));
- if (!videoFrame) {
- m_connection->send(Messages::RemoteVideoFrameObjectHeapProxyProcessor::VideoFrameBufferNotFound { identifier }, 0);
- return;
+ std::optional<SharedVideoFrame::Buffer> buffer;
+ if (videoFrame) {
+ buffer = m_sharedVideoFrameWriter.writeBuffer(videoFrame->pixelBuffer(),
+ [&](auto& semaphore) { m_connection->send(Messages::RemoteVideoFrameObjectHeapProxyProcessor::SetSharedVideoFrameSemaphore { semaphore }, 0); },
+ [&](auto& handle) { m_connection->send(Messages::RemoteVideoFrameObjectHeapProxyProcessor::SetSharedVideoFrameMemory { handle }, 0); },
+ canSendIOSurface);
+ // FIXME: We should ASSERT(result) once we support enough pixel buffer types.
}
-
- auto pixelBuffer = videoFrame->pixelBuffer();
- if (!pixelBuffer) {
- m_connection->send(Messages::RemoteVideoFrameObjectHeapProxyProcessor::VideoFrameBufferNotFound { identifier }, 0);
- ASSERT_NOT_REACHED();
- return;
- }
-
- bool result = m_sharedVideoFrameWriter.write(pixelBuffer,
- [&](auto& semaphore) { m_connection->send(Messages::RemoteVideoFrameObjectHeapProxyProcessor::SetSharedVideoFrameSemaphore { semaphore }, 0); },
- [&](auto& handle) { m_connection->send(Messages::RemoteVideoFrameObjectHeapProxyProcessor::SetSharedVideoFrameMemory { handle }, 0); }
- );
- if (!result) {
- // FIXME: We should ASSERT_NOT_REACHED once we support enough pixel buffer types.
- m_connection->send(Messages::RemoteVideoFrameObjectHeapProxyProcessor::VideoFrameBufferNotFound { identifier }, 0);
- return;
- }
- m_connection->send(Messages::RemoteVideoFrameObjectHeapProxyProcessor::NewVideoFrameBuffer { identifier }, 0);
+ m_connection->send(Messages::RemoteVideoFrameObjectHeapProxyProcessor::NewVideoFrameBuffer { identifier, buffer }, 0);
}
void RemoteVideoFrameObjectHeap::pixelBuffer(RemoteVideoFrameReadReference&& read, CompletionHandler<void(RetainPtr<CVPixelBufferRef>)>&& completionHandler)
@@ -136,7 +123,7 @@
{
assertIsCurrent(remoteVideoFrameObjectHeapQueue());
- auto videoFrame = m_heap.retire(WTFMove(read), 0_s);
+ auto videoFrame = get(WTFMove(read));
if (!videoFrame) {
ASSERT_IS_TESTING_IPC();
completionHandler(nullptr);
Modified: trunk/Source/WebKit/GPUProcess/media/RemoteVideoFrameObjectHeap.h (290823 => 290824)
--- trunk/Source/WebKit/GPUProcess/media/RemoteVideoFrameObjectHeap.h 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteVideoFrameObjectHeap.h 2022-03-04 09:15:08 UTC (rev 290824)
@@ -59,7 +59,7 @@
// Messages.
void releaseVideoFrame(RemoteVideoFrameWriteReference&&);
#if PLATFORM(COCOA)
- void getVideoFrameBuffer(RemoteVideoFrameReadReference&&);
+ void getVideoFrameBuffer(RemoteVideoFrameReadReference&&, bool canSendIOSurface);
void pixelBuffer(RemoteVideoFrameReadReference&&, CompletionHandler<void(RetainPtr<CVPixelBufferRef>)>&&);
#endif
Modified: trunk/Source/WebKit/GPUProcess/media/RemoteVideoFrameObjectHeap.messages.in (290823 => 290824)
--- trunk/Source/WebKit/GPUProcess/media/RemoteVideoFrameObjectHeap.messages.in 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteVideoFrameObjectHeap.messages.in 2022-03-04 09:15:08 UTC (rev 290824)
@@ -24,7 +24,7 @@
#if ENABLE(GPU_PROCESS) && ENABLE(VIDEO)
messages -> RemoteVideoFrameObjectHeap NotRefCounted {
#if PLATFORM(COCOA)
- GetVideoFrameBuffer(WebKit::RemoteVideoFrameReadReference read)
+ GetVideoFrameBuffer(WebKit::RemoteVideoFrameReadReference read, bool canSendIOSurface)
PixelBuffer(WebKit::RemoteVideoFrameReadReference read) -> (RetainPtr<CVPixelBufferRef> result) Synchronous
#endif
ReleaseVideoFrame(WebKit::RemoteVideoFrameWriteReference write)
Modified: trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h (290823 => 290824)
--- trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h 2022-03-04 09:15:08 UTC (rev 290824)
@@ -88,7 +88,7 @@
void createEncoder(RTCEncoderIdentifier, const String&, const Vector<std::pair<String, String>>&, bool useLowLatency);
void releaseEncoder(RTCEncoderIdentifier);
void initializeEncoder(RTCEncoderIdentifier, uint16_t width, uint16_t height, unsigned startBitrate, unsigned maxBitrate, unsigned minBitrate, uint32_t maxFramerate);
- void encodeFrame(RTCEncoderIdentifier, WebCore::RemoteVideoSample&&, uint32_t timeStamp, bool shouldEncodeAsKeyFrame, std::optional<RemoteVideoFrameReadReference>);
+ void encodeFrame(RTCEncoderIdentifier, SharedVideoFrame&&, uint32_t timeStamp, bool shouldEncodeAsKeyFrame);
void setEncodeRates(RTCEncoderIdentifier, uint32_t bitRate, uint32_t frameRate);
void setSharedVideoFrameSemaphore(RTCEncoderIdentifier, IPC::Semaphore&&);
void setSharedVideoFrameMemory(RTCEncoderIdentifier, const SharedMemory::IPCHandle&);
Modified: trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in (290823 => 290824)
--- trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in 2022-03-04 09:15:08 UTC (rev 290824)
@@ -34,7 +34,7 @@
CreateEncoder(WebKit::RTCEncoderIdentifier id, String formatName, Vector<std::pair<String, String>> parameters, bool useLowLatency);
ReleaseEncoder(WebKit::RTCEncoderIdentifier id)
InitializeEncoder(WebKit::RTCEncoderIdentifier id, uint16_t width, uint16_t height, unsigned startBitrate, unsigned maxBitrate, unsigned minBitrate, uint32_t maxFramerate)
- EncodeFrame(WebKit::RTCEncoderIdentifier id, WebCore::RemoteVideoSample sample, uint32_t timeStamp, bool shouldEncodeAsKeyFrame, std::optional<WebKit::RemoteVideoFrameReadReference> sampleReference)
+ EncodeFrame(WebKit::RTCEncoderIdentifier id, struct WebKit::SharedVideoFrame buffer, uint32_t timeStamp, bool shouldEncodeAsKeyFrame)
SetEncodeRates(WebKit::RTCEncoderIdentifier id, uint32_t bitRate, uint32_t frameRate)
SetSharedVideoFrameSemaphore(WebKit::RTCEncoderIdentifier id, IPC::Semaphore semaphore)
SetSharedVideoFrameMemory(WebKit::RTCEncoderIdentifier id, WebKit::SharedMemory::IPCHandle storageHandle)
Modified: trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm (290823 => 290824)
--- trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm 2022-03-04 09:15:08 UTC (rev 290824)
@@ -185,7 +185,7 @@
connection->send(Messages::LibWebRTCCodecs::CompletedEncoding { identifier, IPC::DataReference { buffer, size }, info }, 0);
}).get());
webrtc::setLocalEncoderLowLatency(encoder, useLowLatency);
- auto result = m_encoders.add(identifier, Encoder { encoder, nullptr });
+ auto result = m_encoders.add(identifier, Encoder { encoder, makeUnique<SharedVideoFrameReader>(Ref { m_videoFrameObjectHeap }, m_resourceOwner) });
ASSERT_UNUSED(result, result.isNewEntry || isTestingIPC());
m_hasEncodersOrDecoders = true;
}
@@ -237,39 +237,24 @@
return webrtc::kVideoRotation_0;
}
-void LibWebRTCCodecsProxy::encodeFrame(RTCEncoderIdentifier identifier, WebCore::RemoteVideoSample&& sample, uint32_t timeStamp, bool shouldEncodeAsKeyFrame, std::optional<RemoteVideoFrameReadReference> sampleReference)
+void LibWebRTCCodecsProxy::encodeFrame(RTCEncoderIdentifier identifier, SharedVideoFrame&& sharedVideoFrame, uint32_t timeStamp, bool shouldEncodeAsKeyFrame)
{
assertIsCurrent(workQueue());
- RetainPtr<CVPixelBufferRef> pixelBuffer;
- if (sampleReference) {
- auto sample = m_videoFrameObjectHeap->get(WTFMove(*sampleReference));
- if (!sample)
- return;
-
- auto platformSample = sample->platformSample();
- ASSERT(platformSample.type == WebCore::PlatformSample::CMSampleBufferType);
- pixelBuffer = static_cast<CVPixelBufferRef>(PAL::CMSampleBufferGetImageBuffer(platformSample.sample.cmSampleBuffer));
- }
-
auto* encoder = findEncoder(identifier);
if (!encoder) {
ASSERT_IS_TESTING_IPC();
+ // Make sure to read RemoteVideoFrameReadReference to prevent memory leaks.
+ if (std::holds_alternative<RemoteVideoFrameReadReference>(sharedVideoFrame.buffer))
+ m_videoFrameObjectHeap->get(WTFMove(std::get<RemoteVideoFrameReadReference>(sharedVideoFrame.buffer)));
return;
}
+ auto pixelBuffer = encoder->frameReader->readBuffer(WTFMove(sharedVideoFrame.buffer));
+ if (!pixelBuffer)
+ return;
+
#if !PLATFORM(MACCATALYST)
- if (!pixelBuffer) {
- if (sample.surface()) {
- if (auto buffer = WebCore::createCVPixelBuffer(sample.surface()))
- pixelBuffer = WTFMove(*buffer);
- } else if (encoder->frameReader)
- pixelBuffer = encoder->frameReader->read();
-
- if (!pixelBuffer)
- return;
- }
-
- webrtc::encodeLocalEncoderFrame(encoder->webrtcEncoder, pixelBuffer.get(), sample.time().toTimeScale(1000000).timeValue(), timeStamp, toWebRTCVideoRotation(sample.rotation()), shouldEncodeAsKeyFrame);
+ webrtc::encodeLocalEncoderFrame(encoder->webrtcEncoder, pixelBuffer.get(), sharedVideoFrame.time.toTimeScale(1000000).timeValue(), timeStamp, toWebRTCVideoRotation(sharedVideoFrame.rotation), shouldEncodeAsKeyFrame);
#endif
}
@@ -294,8 +279,6 @@
return;
}
- if (!encoder->frameReader)
- encoder->frameReader = makeUnique<SharedVideoFrameReader>(Ref { m_videoFrameObjectHeap }, m_resourceOwner);
encoder->frameReader->setSemaphore(WTFMove(semaphore));
}
@@ -308,8 +291,6 @@
return;
}
- if (!encoder->frameReader)
- encoder->frameReader = makeUnique<SharedVideoFrameReader>(Ref { m_videoFrameObjectHeap }, m_resourceOwner);
encoder->frameReader->setSharedMemory(ipcHandle);
}
Modified: trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj (290823 => 290824)
--- trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj 2022-03-04 09:15:08 UTC (rev 290824)
@@ -4133,6 +4133,7 @@
4131F3E01F98712C0059995A /* WebServiceWorkerFetchTaskClient.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = WebServiceWorkerFetchTaskClient.cpp; sourceTree = "<group>"; };
4135FBCF1F4FB7F20074C47B /* CacheStorageEngineCaches.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CacheStorageEngineCaches.cpp; sourceTree = "<group>"; };
4135FBD01F4FB7F20074C47B /* CacheStorageEngineCaches.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CacheStorageEngineCaches.h; sourceTree = "<group>"; };
+ 413C542227C90F4C002FCDA8 /* RemoteVideoFrameObjectHeap.messages.in */ = {isa = PBXFileReference; lastKnownFileType = text; path = RemoteVideoFrameObjectHeap.messages.in; sourceTree = "<group>"; };
413C8B1325516BFA00E65055 /* LibWebRTCCodecs.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = LibWebRTCCodecs.mm; sourceTree = "<group>"; };
4150A5A023E06C910051264A /* GPUProcessSessionParameters.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = GPUProcessSessionParameters.h; sourceTree = "<group>"; };
41518535222704F5005430C6 /* ServiceWorkerFetchTask.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ServiceWorkerFetchTask.h; sourceTree = "<group>"; };
@@ -6983,6 +6984,7 @@
07E19F0623D4DC880094FFB4 /* RemoteTextTrackProxy.h */,
7B5A3DA827A7DCBC006C6F97 /* RemoteVideoFrameObjectHeap.cpp */,
7B5A3DA727A7DCBC006C6F97 /* RemoteVideoFrameObjectHeap.h */,
+ 413C542227C90F4C002FCDA8 /* RemoteVideoFrameObjectHeap.messages.in */,
071BC58423CC2C0900680D7C /* RemoteVideoTrackProxy.cpp */,
071BC58523CC2C0900680D7C /* RemoteVideoTrackProxy.h */,
07E19F0523D4DC880094FFB4 /* TextTrackPrivateRemoteConfiguration.h */,
Modified: trunk/Source/WebKit/WebProcess/GPU/media/RemoteVideoFrameProxy.cpp (290823 => 290824)
--- trunk/Source/WebKit/WebProcess/GPU/media/RemoteVideoFrameProxy.cpp 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/WebProcess/GPU/media/RemoteVideoFrameProxy.cpp 2022-03-04 09:15:08 UTC (rev 290824)
@@ -107,10 +107,10 @@
auto videoFrameObjectHeapProxy = std::exchange(m_videoFrameObjectHeapProxy, nullptr);
bool canSendSync = isMainRunLoop(); // FIXME: we should be able to sendSync from other threads too.
- // Currently we just key with this, as there is no condition for "has access to IOKit".
- if (WebProcess::singleton().shouldUseRemoteRenderingForWebGL() || !canSendSync) {
+ bool canUseIOSurface = WebProcess::singleton().shouldUseRemoteRenderingForWebGL();
+ if (!canUseIOSurface || !canSendSync) {
BinarySemaphore semaphore;
- videoFrameObjectHeapProxy->getVideoFrameBuffer(*this, [this, &semaphore](auto pixelBuffer) {
+ videoFrameObjectHeapProxy->getVideoFrameBuffer(*this, canUseIOSurface, [this, &semaphore](auto pixelBuffer) {
m_pixelBuffer = WTFMove(pixelBuffer);
semaphore.signal();
});
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp (290823 => 290824)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp 2022-03-04 09:15:08 UTC (rev 290824)
@@ -482,15 +482,6 @@
return 0;
}
-template<typename Buffer>
-bool LibWebRTCCodecs::copySharedVideoFrame(LibWebRTCCodecs::Encoder& encoder, IPC::Connection& connection, Buffer&& frameBuffer)
-{
- return encoder.sharedVideoFrameWriter.write(frameBuffer,
- [&](auto& semaphore) { connection.send(Messages::LibWebRTCCodecsProxy::SetSharedVideoFrameSemaphore { encoder.identifier, semaphore }, 0); },
- [&](auto& handle) { connection.send(Messages::LibWebRTCCodecsProxy::SetSharedVideoFrameMemory { encoder.identifier, handle }, 0); }
- );
-}
-
int32_t LibWebRTCCodecs::encodeFrame(Encoder& encoder, const webrtc::VideoFrame& frame, bool shouldEncodeAsKeyFrame)
{
ASSERT(!isMainRunLoop());
@@ -500,31 +491,14 @@
if (!connection)
return WEBRTC_VIDEO_CODEC_ERROR;
- std::optional<RemoteVideoFrameReadReference> remoteVideoFrameReadReference;
- if (auto* provider = webrtc::videoFrameBufferProvider(frame)) {
- auto* videoFrame = static_cast<VideoFrame*>(provider);
- if (is<RemoteVideoFrameProxy>(videoFrame))
- remoteVideoFrameReadReference = downcast<RemoteVideoFrameProxy>(videoFrame)->newReadReference();
- }
+ auto buffer = encoder.sharedVideoFrameWriter.writeBuffer(frame,
+ [&](auto& semaphore) { encoder.connection->send(Messages::LibWebRTCCodecsProxy::SetSharedVideoFrameSemaphore { encoder.identifier, semaphore }, 0); },
+ [&](auto& handle) { encoder.connection->send(Messages::LibWebRTCCodecsProxy::SetSharedVideoFrameMemory { encoder.identifier, handle }, 0); });
+ if (!buffer)
+ return WEBRTC_VIDEO_CODEC_ERROR;
- RetainPtr<CVPixelBufferRef> buffer;
- if (!remoteVideoFrameReadReference) {
- buffer = adoptCF(webrtc::pixelBufferFromFrame(frame));
- if (!buffer) {
- // buffer is not native, we need to copy to shared video frame.
- if (!copySharedVideoFrame(encoder, *connection, frame))
- return WEBRTC_VIDEO_CODEC_ERROR;
- }
- }
-
- auto sample = RemoteVideoSample::create(buffer.get(), MediaTime(frame.timestamp_us() * 1000, 1000000), toMediaSampleVideoRotation(frame.rotation()), RemoteVideoSample::ShouldCheckForIOSurface::No);
- if (buffer && !sample->surface()) {
- // buffer is not IOSurface, we need to copy to shared video frame.
- if (!copySharedVideoFrame(encoder, *connection, buffer.get()))
- return WEBRTC_VIDEO_CODEC_ERROR;
- }
-
- connection->send(Messages::LibWebRTCCodecsProxy::EncodeFrame { encoder.identifier, *sample, frame.timestamp(), shouldEncodeAsKeyFrame, remoteVideoFrameReadReference }, 0);
+ SharedVideoFrame sharedVideoFrame { MediaTime(frame.timestamp_us() * 1000, 1000000), false, toMediaSampleVideoRotation(frame.rotation()), WTFMove(*buffer) };
+ encoder.connection->send(Messages::LibWebRTCCodecsProxy::EncodeFrame { encoder.identifier, sharedVideoFrame, frame.timestamp(), shouldEncodeAsKeyFrame }, 0);
return WEBRTC_VIDEO_CODEC_OK;
}
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxy.h (290823 => 290824)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxy.h 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxy.h 2022-03-04 09:15:08 UTC (rev 290824)
@@ -49,7 +49,7 @@
~RemoteVideoFrameObjectHeapProxy() = default;
#if PLATFORM(COCOA)
- void getVideoFrameBuffer(const RemoteVideoFrameProxy& proxy, RemoteVideoFrameObjectHeapProxyProcessor::Callback&& callback) { m_processor->getVideoFrameBuffer(proxy, WTFMove(callback)); }
+ void getVideoFrameBuffer(const RemoteVideoFrameProxy& proxy, bool canUseIOSurface, RemoteVideoFrameObjectHeapProxyProcessor::Callback&& callback) { m_processor->getVideoFrameBuffer(proxy, canUseIOSurface, WTFMove(callback)); }
#endif
private:
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.cpp (290823 => 290824)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.cpp 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.cpp 2022-03-04 09:15:08 UTC (rev 290824)
@@ -94,20 +94,17 @@
return m_callbacks.take(identifier);
}
-void RemoteVideoFrameObjectHeapProxyProcessor::videoFrameBufferNotFound(RemoteVideoFrameIdentifier identifier)
+void RemoteVideoFrameObjectHeapProxyProcessor::newVideoFrameBuffer(RemoteVideoFrameIdentifier identifier, std::optional<SharedVideoFrame::Buffer>&& sharedVideoFrameBuffer)
{
+ RetainPtr<CVPixelBufferRef> pixelBuffer;
+ if (sharedVideoFrameBuffer)
+ pixelBuffer = m_sharedVideoFrameReader.readBuffer(WTFMove(*sharedVideoFrameBuffer));
if (auto callback = takeCallback(identifier))
- callback(nullptr);
-}
+ callback(WTFMove(pixelBuffer));
-void RemoteVideoFrameObjectHeapProxyProcessor::newVideoFrameBuffer(RemoteVideoFrameIdentifier identifier)
-{
- auto result = m_sharedVideoFrameReader.read();
- if (auto callback = takeCallback(identifier))
- callback(WTFMove(result));
}
-void RemoteVideoFrameObjectHeapProxyProcessor::getVideoFrameBuffer(const RemoteVideoFrameProxy& frame, Callback&& callback)
+void RemoteVideoFrameObjectHeapProxyProcessor::getVideoFrameBuffer(const RemoteVideoFrameProxy& frame, bool canUseIOSurface, Callback&& callback)
{
{
Locker lock(m_callbacksLock);
@@ -116,10 +113,10 @@
}
Locker lock(m_connectionLock);
if (!m_connectionID) {
- videoFrameBufferNotFound(frame.identifier());
+ takeCallback(frame.identifier())(nullptr);
return;
}
- IPC::Connection::send(m_connectionID, Messages::RemoteVideoFrameObjectHeap::GetVideoFrameBuffer(frame.newReadReference()), 0, IPC::SendOption::DispatchMessageEvenWhenWaitingForSyncReply);
+ IPC::Connection::send(m_connectionID, Messages::RemoteVideoFrameObjectHeap::GetVideoFrameBuffer(frame.newReadReference(), canUseIOSurface), 0, IPC::SendOption::DispatchMessageEvenWhenWaitingForSyncReply);
}
}
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.h (290823 => 290824)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.h 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.h 2022-03-04 09:15:08 UTC (rev 290824)
@@ -55,7 +55,7 @@
~RemoteVideoFrameObjectHeapProxyProcessor();
using Callback = Function<void(RetainPtr<CVPixelBufferRef>&&)>;
- void getVideoFrameBuffer(const RemoteVideoFrameProxy&, Callback&&);
+ void getVideoFrameBuffer(const RemoteVideoFrameProxy&, bool canUseIOSurfce, Callback&&);
private:
explicit RemoteVideoFrameObjectHeapProxyProcessor(GPUProcessConnection&);
@@ -65,8 +65,7 @@
// Messages
void setSharedVideoFrameSemaphore(IPC::Semaphore&&);
void setSharedVideoFrameMemory(const SharedMemory::IPCHandle&);
- void videoFrameBufferNotFound(RemoteVideoFrameIdentifier);
- void newVideoFrameBuffer(RemoteVideoFrameIdentifier);
+ void newVideoFrameBuffer(RemoteVideoFrameIdentifier, std::optional<SharedVideoFrame::Buffer>&&);
// GPUProcessConnection::Client
void gpuProcessConnectionDidClose(GPUProcessConnection&);
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.messages.in (290823 => 290824)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.messages.in 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/RemoteVideoFrameObjectHeapProxyProcessor.messages.in 2022-03-04 09:15:08 UTC (rev 290824)
@@ -25,8 +25,7 @@
messages -> RemoteVideoFrameObjectHeapProxyProcessor {
SetSharedVideoFrameSemaphore(IPC::Semaphore semaphore)
SetSharedVideoFrameMemory(WebKit::SharedMemory::IPCHandle storageHandle)
- VideoFrameBufferNotFound(WebKit::RemoteVideoFrameIdentifier identifier)
- NewVideoFrameBuffer(WebKit::RemoteVideoFrameIdentifier identifier)
+ NewVideoFrameBuffer(WebKit::RemoteVideoFrameIdentifier identifier, std::optional<WebKit::SharedVideoFrame::Buffer> frame)
}
#endif
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.cpp (290823 => 290824)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.cpp 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.cpp 2022-03-04 09:15:08 UTC (rev 290824)
@@ -37,6 +37,10 @@
#include <WebCore/SharedVideoFrameInfo.h>
#include <wtf/Scope.h>
+#if USE(LIBWEBRTC)
+#include <webrtc/sdk/WebKit/WebKitUtilities.h>
+#endif
+
#include <pal/cf/CoreMediaSoftLink.h>
#include <WebCore/CoreVideoSoftLink.h>
@@ -90,41 +94,55 @@
std::optional<SharedVideoFrame> SharedVideoFrameWriter::write(MediaSample& frame, const Function<void(IPC::Semaphore&)>& newSemaphoreCallback, const Function<void(const SharedMemory::IPCHandle&)>& newMemoryCallback)
{
- SharedVideoFrame sharedVideoFrame { frame.presentationTime(), frame.videoMirrored(), frame.videoRotation(), nullptr };
- if (is<RemoteVideoFrameProxy>(frame)) {
- sharedVideoFrame.buffer = downcast<RemoteVideoFrameProxy>(frame).newReadReference();
- return sharedVideoFrame;
- }
- if (is<MediaSampleAVFObjC>(frame)) {
- if (auto pixelBuffer = downcast<MediaSampleAVFObjC>(frame).pixelBuffer()) {
- if (auto surface = CVPixelBufferGetIOSurface(pixelBuffer)) {
- sharedVideoFrame.buffer = MachSendRight::adopt(IOSurfaceCreateMachPort(surface));
- return sharedVideoFrame;
- }
- }
- }
- if (!write(frame.pixelBuffer(), newSemaphoreCallback, newMemoryCallback))
+ auto buffer = writeBuffer(frame, newSemaphoreCallback, newMemoryCallback);
+ if (!buffer)
return { };
- return sharedVideoFrame;
+ return SharedVideoFrame { frame.presentationTime(), frame.videoMirrored(), frame.videoRotation(), WTFMove(*buffer) };
}
-bool SharedVideoFrameWriter::write(CVPixelBufferRef pixelBuffer, const Function<void(IPC::Semaphore&)>& newSemaphoreCallback, const Function<void(const SharedMemory::IPCHandle&)>& newMemoryCallback)
+std::optional<SharedVideoFrame::Buffer> SharedVideoFrameWriter::writeBuffer(MediaSample& frame, const Function<void(IPC::Semaphore&)>& newSemaphoreCallback, const Function<void(const SharedMemory::IPCHandle&)>& newMemoryCallback)
{
+ if (is<RemoteVideoFrameProxy>(frame))
+ return downcast<RemoteVideoFrameProxy>(frame).newReadReference();
+
+ return writeBuffer(frame.pixelBuffer(), newSemaphoreCallback, newMemoryCallback);
+}
+
+std::optional<SharedVideoFrame::Buffer> SharedVideoFrameWriter::writeBuffer(CVPixelBufferRef pixelBuffer, const Function<void(IPC::Semaphore&)>& newSemaphoreCallback, const Function<void(const SharedMemory::IPCHandle&)>& newMemoryCallback, bool canUseIOSurface)
+{
+ if (!pixelBuffer)
+ return { };
+
+ if (canUseIOSurface) {
+ if (auto surface = CVPixelBufferGetIOSurface(pixelBuffer))
+ return MachSendRight::adopt(IOSurfaceCreateMachPort(surface));
+ }
+
auto info = SharedVideoFrameInfo::fromCVPixelBuffer(pixelBuffer);
if (!prepareWriting(info, newSemaphoreCallback, newMemoryCallback))
- return false;
+ return { };
- return info.writePixelBuffer(pixelBuffer, static_cast<uint8_t*>(m_storage->data()));
+ if (!info.writePixelBuffer(pixelBuffer, static_cast<uint8_t*>(m_storage->data())))
+ return { };
+ return nullptr;
}
#if USE(LIBWEBRTC)
-bool SharedVideoFrameWriter::write(const webrtc::VideoFrame& frame, const Function<void(IPC::Semaphore&)>& newSemaphoreCallback, const Function<void(const SharedMemory::IPCHandle&)>& newMemoryCallback)
+std::optional<SharedVideoFrame::Buffer> SharedVideoFrameWriter::writeBuffer(const webrtc::VideoFrame& frame, const Function<void(IPC::Semaphore&)>& newSemaphoreCallback, const Function<void(const SharedMemory::IPCHandle&)>& newMemoryCallback)
{
+ if (auto* provider = webrtc::videoFrameBufferProvider(frame))
+ return writeBuffer(*static_cast<VideoFrame*>(provider), newSemaphoreCallback, newMemoryCallback);
+
+ if (auto pixelBuffer = adoptCF(webrtc::pixelBufferFromFrame(frame)))
+ return writeBuffer(pixelBuffer.get(), newSemaphoreCallback, newMemoryCallback);
+
auto info = SharedVideoFrameInfo::fromVideoFrame(frame);
if (!prepareWriting(info, newSemaphoreCallback, newMemoryCallback))
return { };
- return info.writeVideoFrame(frame, static_cast<uint8_t*>(m_storage->data()));
+ if (!info.writeVideoFrame(frame, static_cast<uint8_t*>(m_storage->data())))
+ return { };
+ return nullptr;
}
#endif
@@ -143,7 +161,7 @@
SharedVideoFrameReader::SharedVideoFrameReader() = default;
-RetainPtr<CVPixelBufferRef> SharedVideoFrameReader::read()
+RetainPtr<CVPixelBufferRef> SharedVideoFrameReader::readBufferFromSharedMemory()
{
if (!m_storage)
return { };
@@ -169,9 +187,9 @@
return result;
}
-RefPtr<MediaSample> SharedVideoFrameReader::read(SharedVideoFrame&& sharedVideoFrame)
+RetainPtr<CVPixelBufferRef> SharedVideoFrameReader::readBuffer(SharedVideoFrame::Buffer&& buffer)
{
- auto pixelBuffer = switchOn(WTFMove(sharedVideoFrame.buffer),
+ return switchOn(WTFMove(buffer),
[this](RemoteVideoFrameReadReference&& reference) -> RetainPtr<CVPixelBufferRef> {
ASSERT(m_objectHeap);
if (!m_objectHeap)
@@ -188,7 +206,7 @@
return nullptr;
return WebCore::createCVPixelBuffer(surface->surface()).value_or(nullptr);
}, [this](std::nullptr_t representation) -> RetainPtr<CVPixelBufferRef> {
- return read();
+ return readBufferFromSharedMemory();
}, [this](IntSize size) -> RetainPtr<CVPixelBufferRef> {
if (m_blackFrameSize != size) {
m_blackFrameSize = size;
@@ -198,7 +216,11 @@
}
return m_blackFrame.get();
});
+}
+RefPtr<MediaSample> SharedVideoFrameReader::read(SharedVideoFrame&& sharedVideoFrame)
+{
+ auto pixelBuffer = readBuffer(WTFMove(sharedVideoFrame.buffer));
if (!pixelBuffer)
return nullptr;
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.h (290823 => 290824)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.h 2022-03-04 08:53:38 UTC (rev 290823)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.h 2022-03-04 09:15:08 UTC (rev 290824)
@@ -55,7 +55,8 @@
MediaTime time;
bool mirrored { false };
WebCore::MediaSample::VideoRotation rotation { WebCore::MediaSample::VideoRotation::None };
- std::variant<std::nullptr_t, RemoteVideoFrameReadReference, MachSendRight, WebCore::IntSize> buffer;
+ using Buffer = std::variant<std::nullptr_t, RemoteVideoFrameReadReference, MachSendRight, WebCore::IntSize>;
+ Buffer buffer;
template<class Encoder> void encode(Encoder&) const;
template<class Decoder> static std::optional<SharedVideoFrame> decode(Decoder&);
@@ -67,9 +68,9 @@
SharedVideoFrameWriter();
std::optional<SharedVideoFrame> write(WebCore::MediaSample&, const Function<void(IPC::Semaphore&)>&, const Function<void(const SharedMemory::IPCHandle&)>&);
- bool write(CVPixelBufferRef, const Function<void(IPC::Semaphore&)>&, const Function<void(const SharedMemory::IPCHandle&)>&);
+ std::optional<SharedVideoFrame::Buffer> writeBuffer(CVPixelBufferRef, const Function<void(IPC::Semaphore&)>&, const Function<void(const SharedMemory::IPCHandle&)>&, bool canSendIOSurface = true);
#if USE(LIBWEBRTC)
- bool write(const webrtc::VideoFrame&, const Function<void(IPC::Semaphore&)>&, const Function<void(const SharedMemory::IPCHandle&)>&);
+ std::optional<SharedVideoFrame::Buffer> writeBuffer(const webrtc::VideoFrame&, const Function<void(IPC::Semaphore&)>&, const Function<void(const SharedMemory::IPCHandle&)>&);
#endif
void disable();
@@ -77,6 +78,7 @@
bool wait(const Function<void(IPC::Semaphore&)>&);
bool allocateStorage(size_t, const Function<void(const SharedMemory::IPCHandle&)>&);
bool prepareWriting(const WebCore::SharedVideoFrameInfo&, const Function<void(IPC::Semaphore&)>&, const Function<void(const SharedMemory::IPCHandle&)>&);
+ std::optional<SharedVideoFrame::Buffer> writeBuffer(WebCore::MediaSample&, const Function<void(IPC::Semaphore&)>&, const Function<void(const SharedMemory::IPCHandle&)>&);
UniqueRef<IPC::Semaphore> m_semaphore;
RefPtr<SharedMemory> m_storage;
@@ -94,11 +96,12 @@
void setSemaphore(IPC::Semaphore&& semaphore) { m_semaphore = WTFMove(semaphore); }
bool setSharedMemory(const SharedMemory::IPCHandle&);
- RetainPtr<CVPixelBufferRef> read();
RefPtr<WebCore::MediaSample> read(SharedVideoFrame&&);
+ RetainPtr<CVPixelBufferRef> readBuffer(SharedVideoFrame::Buffer&&);
private:
CVPixelBufferPoolRef pixelBufferPool(const WebCore::SharedVideoFrameInfo&);
+ RetainPtr<CVPixelBufferRef> readBufferFromSharedMemory();
RefPtr<RemoteVideoFrameObjectHeap> m_objectHeap;
WebCore::ProcessIdentity m_resourceOwner;
