Diff
Modified: trunk/Source/ThirdParty/libwebrtc/ChangeLog (288950 => 288951)
--- trunk/Source/ThirdParty/libwebrtc/ChangeLog 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/ThirdParty/libwebrtc/ChangeLog 2022-02-02 13:20:58 UTC (rev 288951)
@@ -1,3 +1,19 @@
+2022-02-02 Youenn Fablet <[email protected]>
+
+ LibWebRTCCodecs should not need to create IOSurfaces
+ https://bugs.webkit.org/show_bug.cgi?id=235951
+ <rdar://problem/88326654>
+
+ Reviewed by Eric Carlson.
+
+ Introduce helper routines to copy a webrtc VideoFrame into a memory buffer and get a CVPixelBuffer from a webrtc VideoFrame.
+
+ * Configurations/libwebrtc.iOS.exp:
+ * Configurations/libwebrtc.iOSsim.exp:
+ * Configurations/libwebrtc.mac.exp:
+ * Source/webrtc/sdk/WebKit/WebKitUtilities.h:
+ * Source/webrtc/sdk/WebKit/WebKitUtilities.mm:
+
2022-01-27 Michael Saboff <[email protected]>
com.apple.WebKit.WebAuthn.xpc fails to build with system content path
Modified: trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp (288950 => 288951)
--- trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp 2022-02-02 13:20:58 UTC (rev 288951)
@@ -330,4 +330,5 @@
__ZN6webrtc32webrtc_sequence_checker_internal19SequenceCheckerImplC2Ev
__ZNK3rtc14RTCCertificate17GetSSLCertificateEv
__ZTVN6webrtc30WrappingAsyncDnsResolverResultE
-
+__ZN6webrtc20pixelBufferFromFrameERKNS_10VideoFrameE
+__ZN6webrtc14copyVideoFrameERKNS_10VideoFrameEPh
Modified: trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp (288950 => 288951)
--- trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp 2022-02-02 13:20:58 UTC (rev 288951)
@@ -330,4 +330,5 @@
__ZN6webrtc32webrtc_sequence_checker_internal19SequenceCheckerImplC2Ev
__ZNK3rtc14RTCCertificate17GetSSLCertificateEv
__ZTVN6webrtc30WrappingAsyncDnsResolverResultE
-
+__ZN6webrtc20pixelBufferFromFrameERKNS_10VideoFrameE
+__ZN6webrtc14copyVideoFrameERKNS_10VideoFrameEPh
Modified: trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp (288950 => 288951)
--- trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp 2022-02-02 13:20:58 UTC (rev 288951)
@@ -330,4 +330,5 @@
__ZN6webrtc32webrtc_sequence_checker_internal19SequenceCheckerImplC2Ev
__ZNK3rtc14RTCCertificate17GetSSLCertificateEv
__ZTVN6webrtc30WrappingAsyncDnsResolverResultE
-
+__ZN6webrtc20pixelBufferFromFrameERKNS_10VideoFrameE
+__ZN6webrtc14copyVideoFrameERKNS_10VideoFrameEPh
Modified: trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.h (288950 => 288951)
--- trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.h 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.h 2022-02-02 13:20:58 UTC (rev 288951)
@@ -48,6 +48,8 @@
enum class BufferType { I420, I010 };
CVPixelBufferRef createPixelBufferFromFrame(const VideoFrame&, const std::function<CVPixelBufferRef(size_t, size_t, BufferType)>& createPixelBuffer) CF_RETURNS_RETAINED;
+CVPixelBufferRef pixelBufferFromFrame(const VideoFrame&) CF_RETURNS_RETAINED;
rtc::scoped_refptr<webrtc::VideoFrameBuffer> pixelBufferToFrame(CVPixelBufferRef);
+bool copyVideoFrame(const VideoFrame&, uint8_t*);
}
Modified: trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.mm (288950 => 288951)
--- trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.mm 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.mm 2022-02-02 13:20:58 UTC (rev 288951)
@@ -175,4 +175,52 @@
return CVPixelBufferRetain(rtcPixelBuffer.pixelBuffer);
}
+CVPixelBufferRef pixelBufferFromFrame(const VideoFrame& frame)
+{
+ auto buffer = frame.video_frame_buffer();
+ if (buffer->type() != VideoFrameBuffer::Type::kNative)
+ return nullptr;
+
+ auto *frameBuffer = static_cast<ObjCFrameBuffer*>(buffer.get())->wrapped_frame_buffer();
+ if (![frameBuffer isKindOfClass:[RTCCVPixelBuffer class]])
+ return nullptr;
+
+ auto *rtcPixelBuffer = (RTCCVPixelBuffer *)frameBuffer;
+ return CVPixelBufferRetain(rtcPixelBuffer.pixelBuffer);
}
+
+bool copyVideoFrame(const VideoFrame& frame, uint8_t* data)
+{
+ auto buffer = frame.video_frame_buffer();
+ if (buffer->type() == VideoFrameBuffer::Type::kNative)
+ return false;
+
+ auto type = buffer->type();
+ if (type == VideoFrameBuffer::Type::kI420) {
+ auto* i420Frame = buffer->GetI420();
+ auto* dataY = data;
+ auto strideY = i420Frame->width();
+ auto strideUV = i420Frame->width() / 2;
+ auto* dataUV = data + (i420Frame->width() * i420Frame->height());
+ return !libyuv::I420ToNV12(i420Frame->DataY(), i420Frame->StrideY(),
+ i420Frame->DataU(), i420Frame->StrideU(),
+ i420Frame->DataV(), i420Frame->StrideV(),
+ dataY, strideY, dataUV, strideUV,
+ i420Frame->width(), i420Frame->height());
+ }
+ if (type == VideoFrameBuffer::Type::kI010) {
+ auto* i010Frame = buffer->GetI010();
+ auto* dataY = reinterpret_cast<uint16_t*>(data);
+ auto strideY = i010Frame->width();
+ auto strideUV = i010Frame->width() / 2;
+ auto* dataUV = dataY + (i010Frame->width() * i010Frame->height());
+ return !libyuv::I010ToP010(i010Frame->DataY(), i010Frame->StrideY(),
+ i010Frame->DataU(), i010Frame->StrideU(),
+ i010Frame->DataV(), i010Frame->StrideV(),
+ dataY, strideY, dataUV, strideUV,
+ i010Frame->width(), i010Frame->height());
+ }
+ return false;
+}
+
+}
Modified: trunk/Source/WebCore/ChangeLog (288950 => 288951)
--- trunk/Source/WebCore/ChangeLog 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebCore/ChangeLog 2022-02-02 13:20:58 UTC (rev 288951)
@@ -1,5 +1,26 @@
2022-02-02 Youenn Fablet <[email protected]>
+ LibWebRTCCodecs should not need to create IOSurfaces
+ https://bugs.webkit.org/show_bug.cgi?id=235951
+ <rdar://problem/88326654>
+
+ Reviewed by Eric Carlson.
+
+ Introduce SharedVideoFrameInfo as a helper class to serialize a CVPixelBuffer into memory and create a CVPixelBuffer from reading memory.
+
+ Covered by existing webrtc tests.
+
+ * SourcesCocoa.txt:
+ * WebCore.xcodeproj/project.pbxproj:
+ * platform/cocoa/SharedVideoFrameInfo.h: Added.
+ * platform/cocoa/SharedVideoFrameInfo.mm: Added.
+ * platform/cocoa/CoreVideoSoftLink.cpp:
+ * platform/cocoa/CoreVideoSoftLink.h:
+ * platform/graphics/RemoteVideoSample.cpp:
+ * platform/graphics/RemoteVideoSample.h:
+
+2022-02-02 Youenn Fablet <[email protected]>
+
Clarify that some UUID routines are dedicated to UUID v4
https://bugs.webkit.org/show_bug.cgi?id=235430
Modified: trunk/Source/WebCore/SourcesCocoa.txt (288950 => 288951)
--- trunk/Source/WebCore/SourcesCocoa.txt 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebCore/SourcesCocoa.txt 2022-02-02 13:20:58 UTC (rev 288951)
@@ -268,6 +268,7 @@
platform/cocoa/PowerSourceNotifier.mm
platform/cocoa/RemoteCommandListenerCocoa.mm
platform/cocoa/RuntimeApplicationChecksCocoa.mm
+platform/cocoa/SharedVideoFrameUtilities.mm @no-unify
platform/cocoa/SearchPopupMenuCocoa.mm
platform/cocoa/SharedBufferCocoa.mm
platform/cocoa/SystemVersion.mm
Modified: trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj (288950 => 288951)
--- trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2022-02-02 13:20:58 UTC (rev 288951)
@@ -1211,10 +1211,12 @@
41D129DB1F3D143800D15E47 /* FetchHeaders.h in Headers */ = {isa = PBXBuildFile; fileRef = 41F54F831C50C4F600338488 /* FetchHeaders.h */; settings = {ATTRIBUTES = (Private, ); }; };
41D28D0D2139E05800F4206F /* LibWebRTCStatsCollector.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 41D28D0B2139E01D00F4206F /* LibWebRTCStatsCollector.cpp */; };
41D41C672256874F00697942 /* ServiceWorkerInternals.mm in Sources */ = {isa = PBXBuildFile; fileRef = 41D41C652256859200697942 /* ServiceWorkerInternals.mm */; };
+ 41DC04FE27A83C93008CF968 /* SharedVideoFrameInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 41DC04FB27A83C92008CF968 /* SharedVideoFrameInfo.h */; settings = {ATTRIBUTES = (Private, ); }; };
41DE7C7C222DA14300532B65 /* StorageQuotaManager.h in Headers */ = {isa = PBXBuildFile; fileRef = 41DE7C7B222DA13E00532B65 /* StorageQuotaManager.h */; settings = {ATTRIBUTES = (Private, ); }; };
41DEEFB02719B29000CB8D74 /* VideoFrameRequestCallback.h in Headers */ = {isa = PBXBuildFile; fileRef = 41DEEFAC2719B28800CB8D74 /* VideoFrameRequestCallback.h */; settings = {ATTRIBUTES = (Private, ); }; };
41DEEFB62719BA1A00CB8D74 /* VideoFrameMetadata.h in Headers */ = {isa = PBXBuildFile; fileRef = 41DEEFB42719BA1900CB8D74 /* VideoFrameMetadata.h */; settings = {ATTRIBUTES = (Private, ); }; };
41DEFCB61E56C1BD000D9E5F /* JSDOMMapLike.h in Headers */ = {isa = PBXBuildFile; fileRef = 41DEFCB41E56C1B9000D9E5F /* JSDOMMapLike.h */; };
+ 41DF2D5E27A9452A00F25A21 /* SharedVideoFrameInfo.mm in Sources */ = {isa = PBXBuildFile; fileRef = 41DC04FD27A83C93008CF968 /* SharedVideoFrameInfo.mm */; };
41E12E9F24FE74E20093FFB4 /* WebSocketIdentifier.h in Headers */ = {isa = PBXBuildFile; fileRef = 41E12E9D24FE74E20093FFB4 /* WebSocketIdentifier.h */; settings = {ATTRIBUTES = (Private, ); }; };
41E1B1D10FF5986900576B3B /* AbstractWorker.h in Headers */ = {isa = PBXBuildFile; fileRef = 41E1B1CB0FF5986900576B3B /* AbstractWorker.h */; };
41E2F199274FAECF00A089EE /* NavigationPreloadManager.h in Headers */ = {isa = PBXBuildFile; fileRef = 41E2F197274FAECB00A089EE /* NavigationPreloadManager.h */; };
@@ -8916,6 +8918,8 @@
41D41C652256859200697942 /* ServiceWorkerInternals.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = ServiceWorkerInternals.mm; sourceTree = "<group>"; };
41D51BB21E4E2E8100131A5B /* LibWebRTCAudioFormat.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = LibWebRTCAudioFormat.h; path = libwebrtc/LibWebRTCAudioFormat.h; sourceTree = "<group>"; };
41D6BD98251B3ADA0055A7B7 /* HTMLMediaElement+AudioOutput.idl */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = "HTMLMediaElement+AudioOutput.idl"; sourceTree = "<group>"; };
+ 41DC04FB27A83C92008CF968 /* SharedVideoFrameInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SharedVideoFrameInfo.h; sourceTree = "<group>"; };
+ 41DC04FD27A83C93008CF968 /* SharedVideoFrameInfo.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = SharedVideoFrameInfo.mm; sourceTree = "<group>"; };
41DE7C7A222DA13D00532B65 /* StorageQuotaManager.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = StorageQuotaManager.cpp; sourceTree = "<group>"; };
41DE7C7B222DA13E00532B65 /* StorageQuotaManager.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StorageQuotaManager.h; sourceTree = "<group>"; };
41DEEFAC2719B28800CB8D74 /* VideoFrameRequestCallback.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = VideoFrameRequestCallback.h; sourceTree = "<group>"; };
@@ -26806,6 +26810,8 @@
0E7058F31BC5CCD70045A507 /* SearchPopupMenuCocoa.h */,
0E7058ED1BC5BC190045A507 /* SearchPopupMenuCocoa.mm */,
1A4A95510B4EDCFF002D8C3C /* SharedBufferCocoa.mm */,
+ 41DC04FB27A83C92008CF968 /* SharedVideoFrameInfo.h */,
+ 41DC04FD27A83C93008CF968 /* SharedVideoFrameInfo.mm */,
C1692DD423D23AE0006E88F7 /* SystemBattery.h */,
C1692DD123D23ABD006E88F7 /* SystemBattery.mm */,
5D5975B119635F1100D00878 /* SystemVersion.h */,
@@ -37171,6 +37177,7 @@
CD36C16B260A65CC00C8C529 /* SharedRoutingArbitrator.h in Headers */,
834DFAD01F7DAE5D00C2725B /* SharedStringHash.h in Headers */,
93309EA3099EB78C0056E581 /* SharedTimer.h in Headers */,
+ 41DC04FE27A83C93008CF968 /* SharedVideoFrameInfo.h in Headers */,
467A68DA275EA99C009B31C5 /* SharedWorkerManager.h in Headers */,
467A68D9275EA998009B31C5 /* SharedWorkerProxy.h in Headers */,
467A68D8275EA995009B31C5 /* SharedWorkerScriptLoader.h in Headers */,
@@ -38646,6 +38653,7 @@
43B85ED418CBEC5200E31AF4 /* SelectorPseudoClassAndCompatibilityElementMap.cpp in Sources */,
26AA0F9E18D2A18B00419381 /* SelectorPseudoElementTypeMap.cpp in Sources */,
7CD58DFD1F9565A800112791 /* Settings.cpp in Sources */,
+ 41DF2D5E27A9452A00F25A21 /* SharedVideoFrameInfo.mm in Sources */,
538F10A81F9022A4005102CE /* SoftLinkLibxslt.cpp in Sources */,
CDC8B5AA18047FF10016E685 /* SourceBufferPrivateAVFObjC.mm in Sources */,
A833C7CA0A2CF06B00D57664 /* SVGNames.cpp in Sources */,
Modified: trunk/Source/WebCore/platform/cocoa/CoreVideoSoftLink.cpp (288950 => 288951)
--- trunk/Source/WebCore/platform/cocoa/CoreVideoSoftLink.cpp 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebCore/platform/cocoa/CoreVideoSoftLink.cpp 2022-02-02 13:20:58 UTC (rev 288951)
@@ -39,7 +39,9 @@
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreVideo, CVBufferGetAttachments, CFDictionaryRef, (CVBufferRef buffer, CVAttachmentMode mode), (buffer, mode))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreVideo, CVPixelBufferGetTypeID, CFTypeID, (), ())
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreVideo, CVPixelBufferGetWidth, size_t, (CVPixelBufferRef pixelBuffer), (pixelBuffer))
+SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreVideo, CVPixelBufferGetWidthOfPlane, size_t, (CVPixelBufferRef pixelBuffer, size_t planeIndex), (pixelBuffer, planeIndex))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreVideo, CVPixelBufferGetHeight, size_t, (CVPixelBufferRef pixelBuffer), (pixelBuffer))
+SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreVideo, CVPixelBufferGetHeightOfPlane, size_t, (CVPixelBufferRef pixelBuffer, size_t planeIndex), (pixelBuffer, planeIndex))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreVideo, CVPixelBufferGetBaseAddress, void*, (CVPixelBufferRef pixelBuffer), (pixelBuffer))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreVideo, CVPixelBufferGetBytesPerRow, size_t, (CVPixelBufferRef pixelBuffer), (pixelBuffer))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreVideo, CVPixelBufferGetBytesPerRowOfPlane, size_t, (CVPixelBufferRef pixelBuffer, size_t planeIndex), (pixelBuffer, planeIndex))
Modified: trunk/Source/WebCore/platform/cocoa/CoreVideoSoftLink.h (288950 => 288951)
--- trunk/Source/WebCore/platform/cocoa/CoreVideoSoftLink.h 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebCore/platform/cocoa/CoreVideoSoftLink.h 2022-02-02 13:20:58 UTC (rev 288951)
@@ -44,8 +44,12 @@
#define CVPixelBufferGetTypeID softLink_CoreVideo_CVPixelBufferGetTypeID
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreVideo, CVPixelBufferGetWidth, size_t, (CVPixelBufferRef pixelBuffer), (pixelBuffer))
#define CVPixelBufferGetWidth softLink_CoreVideo_CVPixelBufferGetWidth
+SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreVideo, CVPixelBufferGetWidthOfPlane, size_t, (CVPixelBufferRef pixelBuffer, size_t planeIndex), (pixelBuffer, planeIndex))
+#define CVPixelBufferGetWidthOfPlane softLink_CoreVideo_CVPixelBufferGetWidthOfPlane
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreVideo, CVPixelBufferGetHeight, size_t, (CVPixelBufferRef pixelBuffer), (pixelBuffer))
#define CVPixelBufferGetHeight softLink_CoreVideo_CVPixelBufferGetHeight
+SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreVideo, CVPixelBufferGetHeightOfPlane, size_t, (CVPixelBufferRef pixelBuffer, size_t planeIndex), (pixelBuffer, planeIndex))
+#define CVPixelBufferGetHeightOfPlane softLink_CoreVideo_CVPixelBufferGetHeightOfPlane
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreVideo, CVPixelBufferGetBaseAddress, void*, (CVPixelBufferRef pixelBuffer), (pixelBuffer))
#define CVPixelBufferGetBaseAddress softLink_CoreVideo_CVPixelBufferGetBaseAddress
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreVideo, CVPixelBufferGetBytesPerRow, size_t, (CVPixelBufferRef pixelBuffer), (pixelBuffer))
Added: trunk/Source/WebCore/platform/cocoa/SharedVideoFrameInfo.h (0 => 288951)
--- trunk/Source/WebCore/platform/cocoa/SharedVideoFrameInfo.h (rev 0)
+++ trunk/Source/WebCore/platform/cocoa/SharedVideoFrameInfo.h 2022-02-02 13:20:58 UTC (rev 288951)
@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if ENABLE(MEDIA_STREAM) && PLATFORM(COCOA)
+
+#include <wtf/RetainPtr.h>
+#include <wtf/Span.h>
+
+typedef struct __CVBuffer* CVPixelBufferRef;
+typedef struct __CVPixelBufferPool* CVPixelBufferPoolRef;
+
+namespace webrtc {
+class VideoFrame;
+}
+
+namespace WebCore {
+
+class SharedVideoFrameInfo {
+public:
+ SharedVideoFrameInfo() = default;
+ SharedVideoFrameInfo(OSType, uint32_t width, uint32_t height, uint32_t bytesPerRow, uint32_t widthPlaneB = 0, uint32_t heightPlaneB = 0, uint32_t bytesPerRowPlaneB = 0);
+
+ WEBCORE_EXPORT void encode(uint8_t*);
+ WEBCORE_EXPORT static std::optional<SharedVideoFrameInfo> decode(Span<const uint8_t>);
+
+ WEBCORE_EXPORT static SharedVideoFrameInfo fromCVPixelBuffer(CVPixelBufferRef);
+ WEBCORE_EXPORT bool writePixelBuffer(CVPixelBufferRef, uint8_t* data);
+
+#if USE(LIBWEBRTC)
+ WEBCORE_EXPORT static SharedVideoFrameInfo fromVideoFrame(const webrtc::VideoFrame&);
+ WEBCORE_EXPORT bool writeVideoFrame(const webrtc::VideoFrame&, uint8_t* data);
+#endif
+
+ WEBCORE_EXPORT size_t storageSize() const;
+
+ WEBCORE_EXPORT RetainPtr<CVPixelBufferRef> createPixelBufferFromMemory(const uint8_t* data, CVPixelBufferPoolRef = nullptr);
+
+ WEBCORE_EXPORT bool isReadWriteSupported() const;
+ WEBCORE_EXPORT RetainPtr<CVPixelBufferPoolRef> createCompatibleBufferPool() const;
+
+ OSType bufferType() const { return m_bufferType; }
+ uint32_t width() const { return m_width; };
+ uint32_t height() const { return m_height; };
+
+private:
+ OSType m_bufferType { 0 };
+ uint32_t m_width { 0 };
+ uint32_t m_height { 0 };
+ uint32_t m_bytesPerRow { 0 };
+ uint32_t m_widthPlaneB { 0 };
+ uint32_t m_heightPlaneB { 0 };
+ uint32_t m_bytesPerRowPlaneB { 0 };
+};
+
+
+static constexpr size_t SharedVideoFrameInfoEncodingLength = sizeof(SharedVideoFrameInfo);
+
+inline SharedVideoFrameInfo::SharedVideoFrameInfo(OSType bufferType, uint32_t width, uint32_t height, uint32_t bytesPerRow, uint32_t widthPlaneB, uint32_t heightPlaneB, uint32_t bytesPerRowPlaneB)
+ : m_bufferType(bufferType)
+ , m_width(width)
+ , m_height(height)
+ , m_bytesPerRow(bytesPerRow)
+ , m_widthPlaneB(widthPlaneB)
+ , m_heightPlaneB(heightPlaneB)
+ , m_bytesPerRowPlaneB(bytesPerRowPlaneB)
+{
+}
+
+}
+
+#endif // ENABLE(MEDIA_STREAM)
Added: trunk/Source/WebCore/platform/cocoa/SharedVideoFrameInfo.mm (0 => 288951)
--- trunk/Source/WebCore/platform/cocoa/SharedVideoFrameInfo.mm (rev 0)
+++ trunk/Source/WebCore/platform/cocoa/SharedVideoFrameInfo.mm 2022-02-02 13:20:58 UTC (rev 288951)
@@ -0,0 +1,299 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "SharedVideoFrameInfo.h"
+
+#if ENABLE(MEDIA_STREAM) && PLATFORM(COCOA)
+
+#include "CVUtilities.h"
+#include "IOSurface.h"
+#include "Logging.h"
+#include <wtf/Scope.h>
+#include <wtf/persistence/PersistentCoders.h>
+
+#if USE(ACCELERATE)
+#include <Accelerate/Accelerate.h>
+#endif
+
+#if USE(LIBWEBRTC)
+
+ALLOW_UNUSED_PARAMETERS_BEGIN
+#include <webrtc/api/video/video_frame.h>
+#include <webrtc/sdk/WebKit/WebKitUtilities.h>
+ALLOW_UNUSED_PARAMETERS_END
+
+#endif // USE(LIBWEBRTC)
+
+#include <pal/cf/CoreMediaSoftLink.h>
+#include "CoreVideoSoftLink.h"
+
+namespace WebCore {
+
+SharedVideoFrameInfo SharedVideoFrameInfo::fromCVPixelBuffer(CVPixelBufferRef pixelBuffer)
+{
+ auto type = CVPixelBufferGetPixelFormatType(pixelBuffer);
+ if (type == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || type == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
+ return { type, static_cast<uint32_t>(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0)), static_cast<uint32_t>(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0)), static_cast<uint32_t>(CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0)), static_cast<uint32_t>(CVPixelBufferGetWidthOfPlane(pixelBuffer, 1)), static_cast<uint32_t>(CVPixelBufferGetHeightOfPlane(pixelBuffer, 1)), static_cast<uint32_t>(CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1)) };
+ return { type, static_cast<uint32_t>(CVPixelBufferGetWidth(pixelBuffer)), static_cast<uint32_t>(CVPixelBufferGetHeight(pixelBuffer)), static_cast<uint32_t>(CVPixelBufferGetBytesPerRow(pixelBuffer)) };
+}
+
+bool SharedVideoFrameInfo::isReadWriteSupported() const
+{
+ return m_bufferType == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
+#if USE(ACCELERATE)
+ || m_bufferType == kCVPixelFormatType_32BGRA
+#endif
+ || m_bufferType == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
+ || m_bufferType == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange;
+}
+
+size_t SharedVideoFrameInfo::storageSize() const
+{
+ return (m_bytesPerRow * m_height) + (m_bytesPerRowPlaneB * m_heightPlaneB) + sizeof(SharedVideoFrameInfo);
+}
+
+void SharedVideoFrameInfo::encode(uint8_t* destination)
+{
+ WTF::Persistence::Encoder encoder;
+ encoder << (uint32_t)m_bufferType;
+ encoder << m_width;
+ encoder << m_height;
+ encoder << m_bytesPerRow;
+ encoder << m_widthPlaneB;
+ encoder << m_heightPlaneB;
+ encoder << m_bytesPerRowPlaneB;
+ ASSERT(sizeof(SharedVideoFrameInfo) == encoder.bufferSize());
+ std::memcpy(destination, encoder.buffer(), encoder.bufferSize());
+}
+
+std::optional<SharedVideoFrameInfo> SharedVideoFrameInfo::decode(Span<const uint8_t> span)
+{
+ WTF::Persistence::Decoder decoder(span);
+
+ std::optional<uint32_t> bufferType;
+ decoder >> bufferType;
+ if (!bufferType)
+ return std::nullopt;
+
+ std::optional<uint32_t> width;
+ decoder >> width;
+ if (!width)
+ return std::nullopt;
+
+ std::optional<uint32_t> height;
+ decoder >> height;
+ if (!height)
+ return std::nullopt;
+
+ std::optional<uint32_t> bytesPerRow;
+ decoder >> bytesPerRow;
+ if (!bytesPerRow)
+ return std::nullopt;
+
+ std::optional<uint32_t> widthPlaneB;
+ decoder >> widthPlaneB;
+ if (!widthPlaneB)
+ return std::nullopt;
+
+ std::optional<uint32_t> heightPlaneB;
+ decoder >> heightPlaneB;
+ if (!heightPlaneB)
+ return std::nullopt;
+
+ std::optional<uint32_t> bytesPerRowPlaneB;
+ decoder >> bytesPerRowPlaneB;
+ if (!bytesPerRowPlaneB)
+ return std::nullopt;
+
+ return SharedVideoFrameInfo { *bufferType, *width, *height, *bytesPerRow , *widthPlaneB, *heightPlaneB, *bytesPerRowPlaneB };
+}
+
+RetainPtr<CVPixelBufferRef> SharedVideoFrameInfo::createPixelBufferFromMemory(const uint8_t* data, CVPixelBufferPoolRef bufferPool)
+{
+ ASSERT(isReadWriteSupported());
+ if (m_bufferType == kCVPixelFormatType_32BGRA) {
+#if USE(ACCELERATE)
+ IntSize size { static_cast<int>(m_width), static_cast<int>(m_height) };
+ auto ioSurface = IOSurface::create(size, DestinationColorSpace::SRGB(), IOSurface::Format::BGRA);
+
+ IOSurface::Locker lock(*ioSurface);
+ vImage_Buffer src;
+ src.width = m_width;
+ src.height = m_height;
+ src.rowBytes = m_bytesPerRow;
+ src.data = ""
+
+ vImage_Buffer dest;
+ dest.width = m_width;
+ dest.height = m_height;
+ dest.rowBytes = ioSurface->bytesPerRow();
+ dest.data = ""
+
+ vImageUnpremultiplyData_BGRA8888(&src, &dest, kvImageNoFlags);
+
+ auto pixelBuffer = WebCore::createCVPixelBuffer(ioSurface->surface());
+ if (!pixelBuffer)
+ return nullptr;
+ return WTFMove(*pixelBuffer);
+#else
+ RELEASE_LOG_ERROR(Media, "createIOSurfaceFromSharedMemory cannot convert to IOSurface");
+ return nullptr;
+#endif
+ }
+
+ CVPixelBufferRef rawPixelBuffer = nullptr;
+ if (bufferPool) {
+ auto status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, bufferPool, &rawPixelBuffer);
+ if (status != noErr || !rawPixelBuffer)
+ return nullptr;
+
+ ASSERT(CVPixelBufferGetWidthOfPlane(rawPixelBuffer, 0) == m_width);
+ ASSERT(CVPixelBufferGetHeightOfPlane(rawPixelBuffer, 0) == m_height);
+ ASSERT(CVPixelBufferGetPixelFormatType(rawPixelBuffer) == m_bufferType);
+ } else {
+ auto status = CVPixelBufferCreate(kCFAllocatorDefault, m_width, m_height, m_bufferType, nullptr, &rawPixelBuffer);
+ if (status != noErr || !rawPixelBuffer)
+ return nullptr;
+ }
+
+ auto pixelBuffer = adoptCF(rawPixelBuffer);
+ auto status = CVPixelBufferLockBaseAddress(rawPixelBuffer, 0);
+ if (status != noErr)
+ return nullptr;
+
+ auto scope = makeScopeExit([&rawPixelBuffer] {
+ CVPixelBufferUnlockBaseAddress(rawPixelBuffer, 0);
+ });
+
+ if (CVPixelBufferGetWidthOfPlane(rawPixelBuffer, 1) != m_widthPlaneB || CVPixelBufferGetHeightOfPlane(rawPixelBuffer, 1) != m_heightPlaneB)
+ return nullptr;
+
+ auto* planeA = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(rawPixelBuffer, 0));
+ uint32_t bytesPerRowPlaneA = CVPixelBufferGetBytesPerRowOfPlane(rawPixelBuffer, 0);
+ for (unsigned i = 0; i < m_height; ++i) {
+ std::memcpy(planeA, data, std::min(bytesPerRowPlaneA, m_bytesPerRow));
+ planeA += bytesPerRowPlaneA;
+ data += m_bytesPerRow;
+ }
+
+ auto* planeB = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(rawPixelBuffer, 1));
+ uint32_t bytesPerRowPlaneB = CVPixelBufferGetBytesPerRowOfPlane(rawPixelBuffer, 1);
+ for (unsigned i = 0; i < m_heightPlaneB; ++i) {
+ std::memcpy(planeB, data, std::min(bytesPerRowPlaneB, m_bytesPerRowPlaneB));
+ planeB += bytesPerRowPlaneB;
+ data += m_bytesPerRowPlaneB;
+ }
+
+ CVPixelBufferUnlockBaseAddress(rawPixelBuffer, 0);
+ return pixelBuffer;
+}
+
+bool SharedVideoFrameInfo::writePixelBuffer(CVPixelBufferRef pixelBuffer, uint8_t* data)
+{
+ auto result = CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+ if (result != kCVReturnSuccess)
+ return false;
+
+ auto scope = makeScopeExit([&pixelBuffer] {
+ CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+ });
+
+ encode(data);
+ data += sizeof(SharedVideoFrameInfo);
+
+ if (m_bufferType == kCVPixelFormatType_32BGRA) {
+#if USE(ACCELERATE)
+ vImage_Buffer src;
+ src.width = m_width;
+ src.height = m_height;
+ src.rowBytes = m_bytesPerRow;
+ src.data = ""
+
+ vImage_Buffer dest;
+ dest.width = m_width;
+ dest.height = m_height;
+ dest.rowBytes = m_bytesPerRow;
+ dest.data = ""
+
+ vImageUnpremultiplyData_BGRA8888(&src, &dest, kvImageNoFlags);
+
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ const uint8_t *planeA = static_cast<const uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0));
+ const uint8_t *planeB = static_cast<const uint8_t *>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1));
+
+ size_t planeASize = m_height * m_bytesPerRow;
+ std::memcpy(data, planeA, planeASize);
+ size_t planeBSize = m_heightPlaneB * m_bytesPerRowPlaneB;
+ std::memcpy(data + planeASize, planeB, planeBSize);
+
+ return true;
+}
+
+RetainPtr<CVPixelBufferPoolRef> SharedVideoFrameInfo::createCompatibleBufferPool() const
+{
+ auto result = createIOSurfaceCVPixelBufferPool(m_width, m_height, m_bufferType);
+ if (!result)
+ return { };
+ return *result;
+}
+
+#if USE(LIBWEBRTC)
+SharedVideoFrameInfo SharedVideoFrameInfo::fromVideoFrame(const webrtc::VideoFrame& frame)
+{
+ auto buffer = frame.video_frame_buffer();
+ if (buffer->type() == webrtc::VideoFrameBuffer::Type::kNative)
+ return SharedVideoFrameInfo { };
+
+ auto type = buffer->type();
+
+ if (type == webrtc::VideoFrameBuffer::Type::kI420)
+ return SharedVideoFrameInfo { kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
+ static_cast<uint32_t>(frame.width()), static_cast<uint32_t>(frame.height()), static_cast<uint32_t>(frame.width()),
+ static_cast<uint32_t>(frame.width()), static_cast<uint32_t>(frame.height()), static_cast<uint32_t>(frame.width()) / 2 };
+
+ if (type == webrtc::VideoFrameBuffer::Type::kI010)
+ return SharedVideoFrameInfo { kCVPixelFormatType_420YpCbCr10BiPlanarFullRange,
+ static_cast<uint32_t>(frame.width()), static_cast<uint32_t>(frame.height()), static_cast<uint32_t>(frame.width() * 2),
+ static_cast<uint32_t>(frame.width()), static_cast<uint32_t>(frame.height()), static_cast<uint32_t>(frame.width()) };
+
+ return SharedVideoFrameInfo { };
+}
+
+bool SharedVideoFrameInfo::writeVideoFrame(const webrtc::VideoFrame& frame, uint8_t* data)
+{
+ ASSERT(m_bufferType == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || m_bufferType == kCVPixelFormatType_420YpCbCr10BiPlanarFullRange);
+ return webrtc::copyVideoFrame(frame, data);
+}
+#endif
+
+}
+
+#endif // ENABLE(MEDIA_STREAM) && PLATFORM(COCOA)
Modified: trunk/Source/WebCore/platform/graphics/RemoteVideoSample.cpp (288950 => 288951)
--- trunk/Source/WebCore/platform/graphics/RemoteVideoSample.cpp 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebCore/platform/graphics/RemoteVideoSample.cpp 2022-02-02 13:20:58 UTC (rev 288951)
@@ -85,7 +85,7 @@
#endif
}
-std::unique_ptr<RemoteVideoSample> RemoteVideoSample::create(MediaSample& sample)
+std::unique_ptr<RemoteVideoSample> RemoteVideoSample::create(MediaSample& sample, ShouldCheckForIOSurface shouldCheckForIOSurface)
{
ASSERT(sample.platformSample().type == PlatformSample::CMSampleBufferType);
@@ -97,7 +97,7 @@
std::unique_ptr<IOSurface> ioSurface;
auto surface = CVPixelBufferGetIOSurface(imageBuffer.get());
- if (!surface) {
+ if (!surface && shouldCheckForIOSurface == ShouldCheckForIOSurface::Yes) {
// Special case for canvas data that is RGBA, not IOSurface backed.
auto pixelFormatType = CVPixelBufferGetPixelFormatType(imageBuffer.get());
if (pixelFormatType != kCVPixelFormatType_32BGRA) {
@@ -115,10 +115,10 @@
return std::unique_ptr<RemoteVideoSample>(new RemoteVideoSample(surface, WTFMove(imageBuffer), DestinationColorSpace::SRGB(), sample.presentationTime(), sample.videoRotation(), sample.videoMirrored()));
}
-std::unique_ptr<RemoteVideoSample> RemoteVideoSample::create(RetainPtr<CVPixelBufferRef>&& imageBuffer, MediaTime&& presentationTime, MediaSample::VideoRotation rotation)
+std::unique_ptr<RemoteVideoSample> RemoteVideoSample::create(RetainPtr<CVPixelBufferRef>&& imageBuffer, MediaTime&& presentationTime, MediaSample::VideoRotation rotation, ShouldCheckForIOSurface shouldCheckForIOSurface)
{
auto surface = CVPixelBufferGetIOSurface(imageBuffer.get());
- if (!surface) {
+ if (!surface && shouldCheckForIOSurface == ShouldCheckForIOSurface::Yes) {
RELEASE_LOG_ERROR(Media, "RemoteVideoSample::create: CVPixelBufferGetIOSurface returned nullptr");
return nullptr;
}
Modified: trunk/Source/WebCore/platform/graphics/RemoteVideoSample.h (288950 => 288951)
--- trunk/Source/WebCore/platform/graphics/RemoteVideoSample.h 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebCore/platform/graphics/RemoteVideoSample.h 2022-02-02 13:20:58 UTC (rev 288951)
@@ -46,9 +46,12 @@
RemoteVideoSample& operator=(RemoteVideoSample&&) = default;
~RemoteVideoSample() = default;
- WEBCORE_EXPORT static std::unique_ptr<RemoteVideoSample> create(MediaSample&);
- WEBCORE_EXPORT static std::unique_ptr<RemoteVideoSample> create(RetainPtr<CVPixelBufferRef>&&, MediaTime&& presentationTime, MediaSample::VideoRotation = MediaSample::VideoRotation::None);
+ enum class ShouldCheckForIOSurface { No, Yes };
+ WEBCORE_EXPORT static std::unique_ptr<RemoteVideoSample> create(MediaSample&, ShouldCheckForIOSurface = ShouldCheckForIOSurface::Yes);
+ WEBCORE_EXPORT static std::unique_ptr<RemoteVideoSample> create(RetainPtr<CVPixelBufferRef>&&, MediaTime&& presentationTime, MediaSample::VideoRotation = MediaSample::VideoRotation::None, ShouldCheckForIOSurface = ShouldCheckForIOSurface::Yes);
+
WEBCORE_EXPORT IOSurfaceRef surface() const;
+ CVPixelBufferRef imageBuffer() const { return m_imageBuffer.get(); }
void setOwnershipIdentity(const ProcessIdentity&);
Modified: trunk/Source/WebKit/ChangeLog (288950 => 288951)
--- trunk/Source/WebKit/ChangeLog 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebKit/ChangeLog 2022-02-02 13:20:58 UTC (rev 288951)
@@ -1,5 +1,36 @@
2022-02-02 Youenn Fablet <[email protected]>
+ LibWebRTCCodecs should not need to create IOSurfaces
+ https://bugs.webkit.org/show_bug.cgi?id=235951
+ <rdar://problem/88326654>
+
+ Reviewed by Eric Carlson.
+
+ Introduce SharedVideoFrameWriter and SharedVideoFrameReader to support shared memory sending of CVPixelBuffers that are non IOSurface backed.
+ To do so, we make use of SharedVideoFrameInfo from WebCore.
+ What we do is:
+ - When getting a frame, allocate a shared memory that is big enough to copy its data and info in it. Create a semaphore to sync reader and writer.
+ - If we allocate shared memory/semphore, send them over IPC to the SharedVideoFrameReader.
+ - Wait for semaphore to tell us reader is fine for writing.
+ - Write SharedVideoFrameInfo into the shared memory, then copy the raw bytes into the shared memory.
+ - Send through IPC the video frame, no MacSendRight means to read the shared memory.
+ On reader side, for every frame that does not have an IOSurface, we read the shared memory, create an IOSurface from it and notify WebProcess that it can write the next frame.
+
+ In memory buffers are used for black frames at least.
+ webrtc/video-mute.html covers the patch.
+
+ * GPUProcess/webrtc/LibWebRTCCodecsProxy.h:
+ * GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in:
+ * GPUProcess/webrtc/LibWebRTCCodecsProxy.mm:
+ * SourcesCocoa.txt:
+ * WebKit.xcodeproj/project.pbxproj:
+ * WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp:
+ * WebProcess/GPU/webrtc/LibWebRTCCodecs.h:
+ * WebProcess/GPU/webrtc/SharedVideoFrame.cpp: Added.
+ * WebProcess/GPU/webrtc/SharedVideoFrame.h: Added.
+
+2022-02-02 Youenn Fablet <[email protected]>
+
Clarify that some UUID routines are dedicated to UUID v4
https://bugs.webkit.org/show_bug.cgi?id=235430
Modified: trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h (288950 => 288951)
--- trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.h 2022-02-02 13:20:58 UTC (rev 288951)
@@ -31,11 +31,14 @@
#include "DataReference.h"
#include "RTCDecoderIdentifier.h"
#include "RTCEncoderIdentifier.h"
+#include "SharedMemory.h"
+#include "SharedVideoFrame.h"
#include <wtf/Lock.h>
namespace IPC {
class Connection;
class Decoder;
+class Semaphore;
}
namespace WebCore {
@@ -50,6 +53,7 @@
namespace WebKit {
class GPUConnectionToWebProcess;
+class SharedVideoFrameReader;
class LibWebRTCCodecsProxy : public IPC::Connection::ThreadMessageReceiverRefCounted {
WTF_MAKE_FAST_ALLOCATED;
@@ -81,16 +85,21 @@
void initializeEncoder(RTCEncoderIdentifier, uint16_t width, uint16_t height, unsigned startBitrate, unsigned maxBitrate, unsigned minBitrate, uint32_t maxFramerate);
void encodeFrame(RTCEncoderIdentifier, WebCore::RemoteVideoSample&&, uint32_t timeStamp, bool shouldEncodeAsKeyFrame);
void setEncodeRates(RTCEncoderIdentifier, uint32_t bitRate, uint32_t frameRate);
+ void setSharedVideoFrameSemaphore(RTCEncoderIdentifier, IPC::Semaphore&&);
+ void setSharedVideoFrameMemory(RTCEncoderIdentifier, const SharedMemory::IPCHandle&);
void setRTCLoggingLevel(WTFLogLevel);
- CFDictionaryRef ioSurfacePixelBufferCreationOptions(IOSurfaceRef);
-
GPUConnectionToWebProcess& m_gpuConnectionToWebProcess;
+ struct Encoder {
+ webrtc::LocalEncoder webrtcEncoder { nullptr };
+ std::unique_ptr<SharedVideoFrameReader> frameReader;
+ };
+ Encoder* findEncoderWithoutLock(RTCEncoderIdentifier);
+
mutable Lock m_lock;
HashMap<RTCDecoderIdentifier, webrtc::LocalDecoder> m_decoders WTF_GUARDED_BY_LOCK(m_lock); // Only modified on the libWebRTCCodecsQueue but may get accessed from the main thread.
- HashMap<RTCEncoderIdentifier, webrtc::LocalEncoder> m_encoders WTF_GUARDED_BY_LOCK(m_lock); // Only modified on the libWebRTCCodecsQueue but may get accessed from the main thread.
-
+ HashMap<RTCEncoderIdentifier, Encoder> m_encoders WTF_GUARDED_BY_LOCK(m_lock); // Only modified on the libWebRTCCodecsQueue but may get accessed from the main thread.
Ref<WorkQueue> m_queue;
};
Modified: trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in (288950 => 288951)
--- trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.messages.in 2022-02-02 13:20:58 UTC (rev 288951)
@@ -36,6 +36,8 @@
InitializeEncoder(WebKit::RTCEncoderIdentifier id, uint16_t width, uint16_t height, unsigned startBitrate, unsigned maxBitrate, unsigned minBitrate, uint32_t maxFramerate)
EncodeFrame(WebKit::RTCEncoderIdentifier id, WebCore::RemoteVideoSample sample, uint32_t timeStamp, bool shouldEncodeAsKeyFrame)
SetEncodeRates(WebKit::RTCEncoderIdentifier id, uint32_t bitRate, uint32_t frameRate)
+ SetSharedVideoFrameSemaphore(WebKit::RTCEncoderIdentifier id, IPC::Semaphore semaphore)
+ SetSharedVideoFrameMemory(WebKit::RTCEncoderIdentifier id, WebKit::SharedMemory::IPCHandle storageHandle)
SetRTCLoggingLevel(WTFLogLevel level)
}
Modified: trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm (288950 => 288951)
--- trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebKit/GPUProcess/webrtc/LibWebRTCCodecsProxy.mm 2022-02-02 13:20:58 UTC (rev 288951)
@@ -32,6 +32,7 @@
#import "GPUProcess.h"
#import "LibWebRTCCodecsMessages.h"
#import "LibWebRTCCodecsProxyMessages.h"
+#import "SharedVideoFrame.h"
#import "WebCoreArgumentCoders.h"
#import <WebCore/CVUtilities.h>
#import <WebCore/LibWebRTCProvider.h>
@@ -69,8 +70,8 @@
for (auto decoder : decoders.values())
webrtc::releaseLocalDecoder(decoder);
auto encoders = WTFMove(m_encoders);
- for (auto encoder : encoders.values())
- webrtc::releaseLocalEncoder(encoder);
+ for (auto& encoder : encoders.values())
+ webrtc::releaseLocalEncoder(encoder.webrtcEncoder);
});
}
@@ -159,7 +160,7 @@
connection->send(Messages::LibWebRTCCodecs::CompletedEncoding { identifier, IPC::DataReference { buffer, size }, info }, 0);
}).get());
webrtc::setLocalEncoderLowLatency(encoder, useLowLatency);
- m_encoders.add(identifier, encoder);
+ m_encoders.add(identifier, Encoder { encoder, nullptr });
}
void LibWebRTCCodecsProxy::releaseEncoder(RTCEncoderIdentifier identifier)
@@ -167,23 +168,31 @@
ASSERT(!isMainRunLoop());
Locker locker { m_lock };
ASSERT(m_encoders.contains(identifier));
- if (auto encoder = m_encoders.take(identifier))
- webrtc::releaseLocalEncoder(encoder);
+ auto encoder = m_encoders.take(identifier);
+ if (encoder.webrtcEncoder)
+ webrtc::releaseLocalEncoder(encoder.webrtcEncoder);
}
// For performance reasons, this function accesses m_encoders without locking. This is safe because this function runs on the libWebRTCCodecsQueue
// and m_encoders only get modified on this queue.
-void LibWebRTCCodecsProxy::initializeEncoder(RTCEncoderIdentifier identifier, uint16_t width, uint16_t height, unsigned startBitrate, unsigned maxBitrate, unsigned minBitrate, uint32_t maxFramerate) WTF_IGNORES_THREAD_SAFETY_ANALYSIS
+void LibWebRTCCodecsProxy::initializeEncoder(RTCEncoderIdentifier identifier, uint16_t width, uint16_t height, unsigned startBitrate, unsigned maxBitrate, unsigned minBitrate, uint32_t maxFramerate)
{
- ASSERT(!isMainRunLoop());
- ASSERT(m_encoders.contains(identifier));
- auto encoder = m_encoders.get(identifier);
+ auto* encoder = findEncoderWithoutLock(identifier);
if (!encoder)
return;
- webrtc::initializeLocalEncoder(encoder, width, height, startBitrate, maxBitrate, minBitrate, maxFramerate);
+ webrtc::initializeLocalEncoder(encoder->webrtcEncoder, width, height, startBitrate, maxBitrate, minBitrate, maxFramerate);
}
+LibWebRTCCodecsProxy::Encoder* LibWebRTCCodecsProxy::findEncoderWithoutLock(RTCEncoderIdentifier identifier) WTF_IGNORES_THREAD_SAFETY_ANALYSIS
+{
+ ASSERT(!isMainRunLoop());
+ auto iterator = m_encoders.find(identifier);
+ if (iterator == m_encoders.end())
+ return nullptr;
+ return &iterator->value;
+}
+
static inline webrtc::VideoRotation toWebRTCVideoRotation(WebCore::MediaSample::VideoRotation rotation)
{
switch (rotation) {
@@ -202,36 +211,71 @@
// For performance reasons, this function accesses m_encoders without locking. This is safe because this function runs on the libWebRTCCodecsQueue
// and m_encoders only get modified on this queue.
-void LibWebRTCCodecsProxy::encodeFrame(RTCEncoderIdentifier identifier, WebCore::RemoteVideoSample&& sample, uint32_t timeStamp, bool shouldEncodeAsKeyFrame) WTF_IGNORES_THREAD_SAFETY_ANALYSIS
+void LibWebRTCCodecsProxy::encodeFrame(RTCEncoderIdentifier identifier, WebCore::RemoteVideoSample&& sample, uint32_t timeStamp, bool shouldEncodeAsKeyFrame)
{
- ASSERT(!isMainRunLoop());
- ASSERT(m_encoders.contains(identifier));
- auto encoder = m_encoders.get(identifier);
+ ASSERT(findEncoderWithoutLock(identifier));
+ auto* encoder = findEncoderWithoutLock(identifier);
if (!encoder)
return;
#if !PLATFORM(MACCATALYST)
- if (!sample.surface())
- return;
- auto pixelBuffer = WebCore::createCVPixelBuffer(sample.surface());
+ RetainPtr<CVPixelBufferRef> pixelBuffer;
+ if (sample.surface()) {
+ if (auto buffer = WebCore::createCVPixelBuffer(sample.surface()))
+ pixelBuffer = WTFMove(*buffer);
+ } else if (encoder->frameReader)
+ pixelBuffer = encoder->frameReader->read();
+
if (!pixelBuffer)
return;
- webrtc::encodeLocalEncoderFrame(encoder, pixelBuffer->get(), sample.time().toTimeScale(1000000).timeValue(), timeStamp, toWebRTCVideoRotation(sample.rotation()), shouldEncodeAsKeyFrame);
+
+ webrtc::encodeLocalEncoderFrame(encoder->webrtcEncoder, pixelBuffer.get(), sample.time().toTimeScale(1000000).timeValue(), timeStamp, toWebRTCVideoRotation(sample.rotation()), shouldEncodeAsKeyFrame);
#endif
}
// For performance reasons, this function accesses m_encoders without locking. This is safe because this function runs on the libWebRTCCodecsQueue
// and m_encoders only get modified on this queue.
-void LibWebRTCCodecsProxy::setEncodeRates(RTCEncoderIdentifier identifier, uint32_t bitRate, uint32_t frameRate) WTF_IGNORES_THREAD_SAFETY_ANALYSIS
+void LibWebRTCCodecsProxy::setEncodeRates(RTCEncoderIdentifier identifier, uint32_t bitRate, uint32_t frameRate)
{
- ASSERT(!isMainRunLoop());
- auto encoder = m_encoders.get(identifier);
+ auto* encoder = findEncoderWithoutLock(identifier);
if (!encoder)
return;
- webrtc::setLocalEncoderRates(encoder, bitRate, frameRate);
+ webrtc::setLocalEncoderRates(encoder->webrtcEncoder, bitRate, frameRate);
}
+// For performance reasons, this function accesses m_encoders without locking. This is safe because this function runs on the libWebRTCCodecsQueue
+// and m_encoders only get modified on this queue.
+void LibWebRTCCodecsProxy::setSharedVideoFrameSemaphore(RTCEncoderIdentifier identifier, IPC::Semaphore&& semaphore)
+{
+ ASSERT(findEncoderWithoutLock(identifier));
+ auto* encoder = findEncoderWithoutLock(identifier);
+ if (!encoder)
+ return;
+
+ if (!encoder->frameReader)
+ encoder->frameReader = makeUnique<SharedVideoFrameReader>();
+ encoder->frameReader->setSemaphore(WTFMove(semaphore));
+}
+
+// For performance reasons, this function accesses m_encoders without locking. This is safe because this function runs on the libWebRTCCodecsQueue
+// and m_encoders only get modified on this queue.
+void LibWebRTCCodecsProxy::setSharedVideoFrameMemory(RTCEncoderIdentifier identifier, const SharedMemory::IPCHandle& ipcHandle)
+{
+ ASSERT(findEncoderWithoutLock(identifier));
+ auto* encoder = findEncoderWithoutLock(identifier);
+ if (!encoder)
+ return;
+
+ auto memory = SharedMemory::map(ipcHandle.handle, SharedMemory::Protection::ReadOnly);
+ if (!memory)
+ return;
+
+ if (!encoder->frameReader)
+ encoder->frameReader = makeUnique<SharedVideoFrameReader>();
+ encoder->frameReader->setSharedMemory(memory.releaseNonNull());
+}
+
bool LibWebRTCCodecsProxy::allowsExitUnderMemoryPressure() const
{
ASSERT(isMainRunLoop());
Modified: trunk/Source/WebKit/SourcesCocoa.txt (288950 => 288951)
--- trunk/Source/WebKit/SourcesCocoa.txt 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebKit/SourcesCocoa.txt 2022-02-02 13:20:58 UTC (rev 288951)
@@ -244,6 +244,8 @@
Shared/RemoteLayerTree/RemoteScrollingCoordinatorTransaction.cpp
Shared/RemoteLayerTree/RemoteScrollingUIState.cpp
+WebProcess/GPU/webrtc/SharedVideoFrame.cpp
+
UIProcess/_WKWebViewPrintFormatter.mm
UIProcess/ApplicationStateTracker.mm
UIProcess/HighPerformanceGraphicsUsageSampler.cpp
Modified: trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj (288950 => 288951)
--- trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj 2022-02-02 13:20:58 UTC (rev 288951)
@@ -4157,6 +4157,8 @@
41D129D91F3D101400D15E47 /* WebCacheStorageProvider.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WebCacheStorageProvider.h; sourceTree = "<group>"; };
41D5C6D8238EB20D00B9B3CB /* ServiceWorkerSoftUpdateLoader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ServiceWorkerSoftUpdateLoader.h; sourceTree = "<group>"; };
41D5C6D9238EB20E00B9B3CB /* ServiceWorkerSoftUpdateLoader.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ServiceWorkerSoftUpdateLoader.cpp; sourceTree = "<group>"; };
+ 41DC04F827A816BB008CF968 /* SharedVideoFrame.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SharedVideoFrame.h; sourceTree = "<group>"; };
+ 41DC04F927A816BB008CF968 /* SharedVideoFrame.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = SharedVideoFrame.cpp; sourceTree = "<group>"; };
41DC45941E3D6E1E00B11F51 /* NetworkRTCProvider.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NetworkRTCProvider.h; sourceTree = "<group>"; };
41DC45951E3D6E1E00B11F51 /* NetworkRTCProvider.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = NetworkRTCProvider.cpp; sourceTree = "<group>"; };
41DC45981E3D6ED600B11F51 /* NetworkRTCProvider.messages.in */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = NetworkRTCProvider.messages.in; sourceTree = "<group>"; };
@@ -7214,7 +7216,6 @@
7AFBD36E21E546E3005DBACB /* PersistencyUtils.cpp */,
7AFBD36D21E546E3005DBACB /* PersistencyUtils.h */,
BCC43AB8127B95DC00317F16 /* PlatformPopupMenuData.cpp */,
- BCC43AB9127B95DC00317F16 /* PlatformPopupMenuData.h */,
49ECA41B23FCA4860023358D /* PolicyDecision.h */,
E18C92F312DB9E7100CF2AEB /* PrintInfo.cpp */,
E1CC1B8E12D7EADF00625838 /* PrintInfo.h */,
@@ -7234,6 +7235,7 @@
2D65D196274B8E84009C4101 /* ScrollingAccelerationCurve.cpp */,
2D65D195274B8E84009C4101 /* ScrollingAccelerationCurve.h */,
5C80B3DD23690F100086E6DE /* ServiceWorkerInitializationData.cpp */,
+ BCC43AB9127B95DC00317F16 /* PlatformPopupMenuData.h */,
5C80B3DB23690D8D0086E6DE /* ServiceWorkerInitializationData.h */,
1AFDE6571954A42B00C48FFA /* SessionState.cpp */,
1AFDE6581954A42B00C48FFA /* SessionState.h */,
@@ -9215,6 +9217,8 @@
41FCD6BC23CE031800C62567 /* SampleBufferDisplayLayerIdentifier.h */,
41024FC123CF104500FDF98E /* SampleBufferDisplayLayerManager.cpp */,
41024FC023CF104500FDF98E /* SampleBufferDisplayLayerManager.h */,
+ 41DC04F927A816BB008CF968 /* SharedVideoFrame.cpp */,
+ 41DC04F827A816BB008CF968 /* SharedVideoFrame.h */,
);
path = webrtc;
sourceTree = "<group>";
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp (288950 => 288951)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.cpp 2022-02-02 13:20:58 UTC (rev 288951)
@@ -187,7 +187,7 @@
m_connection->addThreadMessageReceiver(Messages::LibWebRTCCodecs::messageReceiverName(), this);
if (m_loggingLevel)
- m_connection->send(Messages::LibWebRTCCodecsProxy::SetRTCLoggingLevel(*m_loggingLevel), 0);
+ m_connection->send(Messages::LibWebRTCCodecsProxy::SetRTCLoggingLevel { *m_loggingLevel }, 0);
}
// May be called on any thread.
@@ -418,6 +418,22 @@
return 0;
}
+bool LibWebRTCCodecs::copySharedVideoFrame(Encoder& encoder, CVPixelBufferRef pixelBuffer)
+{
+ return encoder.sharedVideoFrameWriter.write(pixelBuffer,
+ [&encoder](auto& semaphore) { encoder.connection->send(Messages::LibWebRTCCodecsProxy::SetSharedVideoFrameSemaphore { encoder.identifier, semaphore }, 0); },
+ [&encoder](auto& handle) { encoder.connection->send(Messages::LibWebRTCCodecsProxy::SetSharedVideoFrameMemory { encoder.identifier, handle }, 0); }
+ );
+}
+
+bool LibWebRTCCodecs::copySharedVideoFrame(Encoder& encoder, const webrtc::VideoFrame& frame)
+{
+ return encoder.sharedVideoFrameWriter.write(frame,
+ [&encoder](auto& semaphore) { encoder.connection->send(Messages::LibWebRTCCodecsProxy::SetSharedVideoFrameSemaphore { encoder.identifier, semaphore }, 0); },
+ [&encoder](auto& handle) { encoder.connection->send(Messages::LibWebRTCCodecsProxy::SetSharedVideoFrameMemory { encoder.identifier, handle }, 0); }
+ );
+}
+
int32_t LibWebRTCCodecs::encodeFrame(Encoder& encoder, const webrtc::VideoFrame& frame, bool shouldEncodeAsKeyFrame)
{
Locker locker { m_connectionLock };
@@ -425,34 +441,18 @@
if (!encoder.connection)
return WEBRTC_VIDEO_CODEC_ERROR;
- auto pixelBuffer = adoptCF(webrtc::createPixelBufferFromFrame(frame, [](size_t width, size_t height, webrtc::BufferType bufferType) -> CVPixelBufferRef {
- OSType poolBufferType;
- switch (bufferType) {
- case webrtc::BufferType::I420:
- poolBufferType = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
- break;
- case webrtc::BufferType::I010:
- poolBufferType = kCVPixelFormatType_420YpCbCr10BiPlanarFullRange;
- }
- auto pixelBufferPool = WebProcess::singleton().libWebRTCCodecs().pixelBufferPool(width, height, poolBufferType);
- if (!pixelBufferPool)
- return nullptr;
-
- return WebCore::createCVPixelBufferFromPool(pixelBufferPool).value_or(nullptr).leakRef();
- }));
-
- if (!pixelBuffer)
- return WEBRTC_VIDEO_CODEC_ERROR;
-
- auto sample = RemoteVideoSample::create(pixelBuffer.get(), MediaTime(frame.timestamp_us() * 1000, 1000000), toMediaSampleVideoRotation(frame.rotation()));
- if (!sample) {
- // FIXME: Optimize this code path, currently we have non BGRA for muted frames at least.
- sample = RemoteVideoSample::create(convertToBGRA(pixelBuffer.get()), MediaTime(frame.timestamp_us() * 1000, 1000000), toMediaSampleVideoRotation(frame.rotation()));
- if (!sample) {
- RELEASE_LOG_ERROR(WebRTC, "Unable to convert remote video sample");
+ auto buffer = adoptCF(webrtc::pixelBufferFromFrame(frame));
+ if (!buffer) {
+ // buffer is not native, we need to copy to shared video frame.
+ if (!copySharedVideoFrame(encoder, frame))
return WEBRTC_VIDEO_CODEC_ERROR;
- }
}
+ auto sample = RemoteVideoSample::create(buffer.get(), MediaTime(frame.timestamp_us() * 1000, 1000000), toMediaSampleVideoRotation(frame.rotation()), RemoteVideoSample::ShouldCheckForIOSurface::No);
+ if (!sample->surface()) {
+ // buffer is not IOSurface, we need to copy to shared video frame.
+ if (!copySharedVideoFrame(encoder, buffer.get()))
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
encoder.connection->send(Messages::LibWebRTCCodecsProxy::EncodeFrame { encoder.identifier, *sample, frame.timestamp(), shouldEncodeAsKeyFrame }, 0);
return WEBRTC_VIDEO_CODEC_OK;
@@ -535,6 +535,7 @@
m_connection->send(Messages::LibWebRTCCodecsProxy::CreateEncoder { encoder->identifier, formatNameFromWebRTCCodecType(encoder->codecType), encoder->parameters, RuntimeEnabledFeatures::sharedFeatures().webRTCH264LowLatencyEncoderEnabled() }, 0);
if (encoder->initializationData)
m_connection->send(Messages::LibWebRTCCodecsProxy::InitializeEncoder { encoder->identifier, encoder->initializationData->width, encoder->initializationData->height, encoder->initializationData->startBitRate, encoder->initializationData->maxBitRate, encoder->initializationData->minBitRate, encoder->initializationData->maxFrameRate }, 0);
+ encoder->sharedVideoFrameWriter = { };
encoder->connection = m_connection.get();
}
for (auto& decoder : m_decoders.values()) {
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.h (288950 => 288951)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.h 2022-02-02 10:30:01 UTC (rev 288950)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/LibWebRTCCodecs.h 2022-02-02 13:20:58 UTC (rev 288951)
@@ -30,9 +30,11 @@
#include "Connection.h"
#include "DataReference.h"
#include "GPUProcessConnection.h"
+#include "IPCSemaphore.h"
#include "MessageReceiver.h"
#include "RTCDecoderIdentifier.h"
#include "RTCEncoderIdentifier.h"
+#include "SharedVideoFrame.h"
#include <WebCore/PixelBufferConformerCV.h>
#include <map>
#include <webrtc/api/video/video_codec_type.h>
@@ -101,6 +103,7 @@
void* encodedImageCallback WTF_GUARDED_BY_LOCK(encodedImageCallbackLock) { nullptr };
Lock encodedImageCallbackLock;
RefPtr<IPC::Connection> connection;
+ SharedVideoFrameWriter sharedVideoFrameWriter;
};
Encoder* createEncoder(Type, const std::map<std::string, std::string>&);
@@ -135,6 +138,9 @@
// GPUProcessConnection::Client
void gpuProcessConnectionDidClose(GPUProcessConnection&);
+ bool copySharedVideoFrame(Encoder&, CVPixelBufferRef);
+ bool copySharedVideoFrame(Encoder&, const webrtc::VideoFrame&);
+
private:
HashMap<RTCDecoderIdentifier, std::unique_ptr<Decoder>> m_decoders;
HashSet<RTCDecoderIdentifier> m_decodingErrors;
Added: trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.cpp (0 => 288951)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.cpp (rev 0)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.cpp 2022-02-02 13:20:58 UTC (rev 288951)
@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "SharedVideoFrame.h"
+
+#if ENABLE(MEDIA_STREAM) && PLATFORM(COCOA)
+
+#include "Logging.h"
+#include <WebCore/SharedVideoFrameInfo.h>
+#include <wtf/Scope.h>
+
+namespace WebKit {
+using namespace WebCore;
+
+bool SharedVideoFrameWriter::wait(const Function<void(IPC::Semaphore&)>& newSemaphoreCallback)
+{
+ if (!m_semaphore) {
+ m_semaphore = makeUnique<IPC::Semaphore>();
+ newSemaphoreCallback(*m_semaphore);
+ return true;
+ }
+ return m_semaphore->wait();
+}
+
+bool SharedVideoFrameWriter::allocateStorage(size_t size, const Function<void(const SharedMemory::IPCHandle&)>& newMemoryCallback)
+{
+ m_storage = SharedMemory::allocate(size);
+ if (!m_storage)
+ return false;
+
+ SharedMemory::Handle handle;
+ m_storage->createHandle(handle, SharedMemory::Protection::ReadOnly);
+
+ auto dataSize = handle.size();
+ newMemoryCallback(SharedMemory::IPCHandle { WTFMove(handle), dataSize });
+ return true;
+}
+
+bool SharedVideoFrameWriter::prepareWriting(const SharedVideoFrameInfo& info, const Function<void(IPC::Semaphore&)>& newSemaphoreCallback, const Function<void(const SharedMemory::IPCHandle&)>& newMemoryCallback)
+{
+ if (!info.isReadWriteSupported())
+ return false;
+
+ if (!wait(newSemaphoreCallback))
+ return false;
+
+ size_t size = info.storageSize();
+ if (!m_storage || m_storage->size() < size) {
+ if (!allocateStorage(size, newMemoryCallback))
+ return false;
+ }
+ return true;
+}
+
+bool SharedVideoFrameWriter::write(CVPixelBufferRef pixelBuffer, const Function<void(IPC::Semaphore&)>& newSemaphoreCallback, const Function<void(const SharedMemory::IPCHandle&)>& newMemoryCallback)
+{
+ auto info = SharedVideoFrameInfo::fromCVPixelBuffer(pixelBuffer);
+ if (!prepareWriting(info, newSemaphoreCallback, newMemoryCallback))
+ return false;
+
+ return info.writePixelBuffer(pixelBuffer, static_cast<uint8_t*>(m_storage->data()));
+}
+
+#if USE(LIBWEBRTC)
+bool SharedVideoFrameWriter::write(const webrtc::VideoFrame& frame, const Function<void(IPC::Semaphore&)>& newSemaphoreCallback, const Function<void(const SharedMemory::IPCHandle&)>& newMemoryCallback)
+{
+ auto info = SharedVideoFrameInfo::fromVideoFrame(frame);
+ if (!prepareWriting(info, newSemaphoreCallback, newMemoryCallback))
+ return false;
+
+ return info.writeVideoFrame(frame, static_cast<uint8_t*>(m_storage->data()));
+}
+#endif
+
+RetainPtr<CVPixelBufferRef> SharedVideoFrameReader::read()
+{
+ if (!m_storage)
+ return { };
+
+ auto scope = makeScopeExit([&] {
+ m_semaphore.signal();
+ });
+
+ auto* data = "" uint8_t*>(m_storage->data());
+ auto info = SharedVideoFrameInfo::decode({ data, m_storage->size() });
+ if (!info)
+ return { };
+
+ if (!info->isReadWriteSupported())
+ return { };
+
+ if (m_storage->size() < info->storageSize())
+ return { };
+
+ return info->createPixelBufferFromMemory(data + SharedVideoFrameInfoEncodingLength, pixelBufferPool(*info));
+}
+
+CVPixelBufferPoolRef SharedVideoFrameReader::pixelBufferPool(const SharedVideoFrameInfo& info)
+{
+ if (m_useIOSurfaceBufferPool == UseIOSurfaceBufferPool::No)
+ return nullptr;
+
+ if (!m_bufferPool || m_bufferPoolType != info.bufferType() || m_bufferPoolWidth != info.width() || m_bufferPoolHeight != info.height()) {
+ m_bufferPoolType = info.bufferType();
+ m_bufferPoolWidth = info.width();
+ m_bufferPoolHeight = info.height();
+ m_bufferPool = info.createCompatibleBufferPool();
+ }
+
+ return m_bufferPool.get();
+}
+
+
+}
+
+#endif // ENABLE(MEDIA_STREAM) && PLATFORM(COCOA)
Added: trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.h (0 => 288951)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.h (rev 0)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/SharedVideoFrame.h 2022-02-02 13:20:58 UTC (rev 288951)
@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2022 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if ENABLE(MEDIA_STREAM) && PLATFORM(COCOA)
+
+#include "IPCSemaphore.h"
+#include "SharedMemory.h"
+#include <wtf/RefPtr.h>
+#include <wtf/RetainPtr.h>
+
+typedef struct __CVBuffer* CVPixelBufferRef;
+typedef struct __CVPixelBufferPool* CVPixelBufferPoolRef;
+
+namespace WebCore {
+class SharedVideoFrameInfo;
+}
+
+namespace webrtc {
+class VideoFrame;
+}
+
+namespace WebKit {
+
+class SharedVideoFrameWriter {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ SharedVideoFrameWriter() = default;
+
+ bool write(CVPixelBufferRef, const Function<void(IPC::Semaphore&)>&, const Function<void(const SharedMemory::IPCHandle&)>&);
+#if USE(LIBWEBRTC)
+ bool write(const webrtc::VideoFrame&, const Function<void(IPC::Semaphore&)>&, const Function<void(const SharedMemory::IPCHandle&)>&);
+#endif
+
+private:
+ bool wait(const Function<void(IPC::Semaphore&)>&);
+ bool allocateStorage(size_t, const Function<void(const SharedMemory::IPCHandle&)>&);
+ bool prepareWriting(const WebCore::SharedVideoFrameInfo&, const Function<void(IPC::Semaphore&)>&, const Function<void(const SharedMemory::IPCHandle&)>&);
+
+ std::unique_ptr<IPC::Semaphore> m_semaphore;
+ RefPtr<SharedMemory> m_storage;
+};
+
+class SharedVideoFrameReader {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ enum class UseIOSurfaceBufferPool { No, Yes };
+ explicit SharedVideoFrameReader(UseIOSurfaceBufferPool = UseIOSurfaceBufferPool::Yes);
+
+ void setSemaphore(IPC::Semaphore&& semaphore) { m_semaphore = WTFMove(semaphore); }
+ void setSharedMemory(Ref<SharedMemory>&& storage) { m_storage = WTFMove(storage); }
+
+ RetainPtr<CVPixelBufferRef> read();
+
+private:
+ CVPixelBufferPoolRef pixelBufferPool(const WebCore::SharedVideoFrameInfo&);
+
+ UseIOSurfaceBufferPool m_useIOSurfaceBufferPool;
+ IPC::Semaphore m_semaphore;
+ RefPtr<SharedMemory> m_storage;
+
+ RetainPtr<CVPixelBufferPoolRef> m_bufferPool;
+ OSType m_bufferPoolType { 0 };
+ uint32_t m_bufferPoolWidth { 0 };
+ uint32_t m_bufferPoolHeight { 0 };
+};
+
+inline SharedVideoFrameReader::SharedVideoFrameReader(UseIOSurfaceBufferPool useIOSurfaceBufferPool)
+ : m_useIOSurfaceBufferPool(useIOSurfaceBufferPool)
+{
+}
+
+
+}
+
+#endif // ENABLE(MEDIA_STREAM)
