Diff
Modified: trunk/Source/ThirdParty/libwebrtc/ChangeLog (236422 => 236423)
--- trunk/Source/ThirdParty/libwebrtc/ChangeLog 2018-09-24 20:12:57 UTC (rev 236422)
+++ trunk/Source/ThirdParty/libwebrtc/ChangeLog 2018-09-24 20:15:17 UTC (rev 236423)
@@ -1,3 +1,27 @@
+2018-09-24 Youenn Fablet <[email protected]>
+
+ Enable conversion of libwebrtc internal frames as CVPixelBuffer
+ https://bugs.webkit.org/show_bug.cgi?id=189892
+
+ Reviewed by Eric Carlson.
+
+ Renamed encoder/decoder factory creation routine.
+ Make pixelBufferFromFrame take a function to create a CVPixelBuffer
+ if the frame does not wrap one.
+ Initialize the CVPixelBuffer with libwebrtc internal frame.
+
+ * Configurations/libwebrtc.iOS.exp:
+ * Configurations/libwebrtc.iOSsim.exp:
+ * Configurations/libwebrtc.mac.exp:
+ * Source/webrtc/sdk/WebKit/WebKitUtilities.h:
+ * Source/webrtc/sdk/WebKit/WebKitUtilities.mm:
+ (webrtc::createWebKitEncoderFactory):
+ (webrtc::createWebKitDecoderFactory):
+ (webrtc::CopyVideoFrameToPixelBuffer):
+ (webrtc::pixelBufferFromFrame):
+ (webrtc::createVideoToolboxEncoderFactory): Deleted.
+ (webrtc::createVideoToolboxDecoderFactory): Deleted.
+
2018-09-21 Thibault Saunier <[email protected]>
[libwebrtc] Allow IP mismatch for local connections on localhost
Modified: trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp (236422 => 236423)
--- trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp 2018-09-24 20:12:57 UTC (rev 236422)
+++ trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOS.exp 2018-09-24 20:15:17 UTC (rev 236423)
@@ -105,10 +105,10 @@
__ZN6webrtc27SessionDescriptionInterface16RemoveCandidatesERKNSt3__16vectorIN7cricket9CandidateENS1_9allocatorIS4_EEEE
__ZNK6webrtc21IceCandidateInterface10server_urlEv
__ZN6webrtc20setApplicationStatusEb
-__ZN6webrtc32createVideoToolboxDecoderFactoryEv
-__ZN6webrtc32createVideoToolboxEncoderFactoryEv
+__ZN6webrtc26createWebKitDecoderFactoryEv
+__ZN6webrtc26createWebKitEncoderFactoryEv
__ZN6webrtc29setH264HardwareEncoderAllowedEb
-__ZN6webrtc20pixelBufferFromFrameERKNS_10VideoFrameE
+__ZN6webrtc20pixelBufferFromFrameERKNS_10VideoFrameERKNSt3__18functionIFP10__CVBuffermmEEE
__ZN6webrtc18pixelBufferToFrameEP10__CVBuffer
__ZN3rtc24BasicPacketSocketFactory19CreateAsyncResolverEv
__ZN3rtc24BasicPacketSocketFactoryC2Ev
Modified: trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp (236422 => 236423)
--- trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp 2018-09-24 20:12:57 UTC (rev 236422)
+++ trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.iOSsim.exp 2018-09-24 20:15:17 UTC (rev 236423)
@@ -105,10 +105,10 @@
__ZN6webrtc27SessionDescriptionInterface16RemoveCandidatesERKNSt3__16vectorIN7cricket9CandidateENS1_9allocatorIS4_EEEE
__ZNK6webrtc21IceCandidateInterface10server_urlEv
__ZN6webrtc20setApplicationStatusEb
-__ZN6webrtc32createVideoToolboxDecoderFactoryEv
-__ZN6webrtc32createVideoToolboxEncoderFactoryEv
+__ZN6webrtc26createWebKitDecoderFactoryEv
+__ZN6webrtc26createWebKitEncoderFactoryEv
__ZN6webrtc29setH264HardwareEncoderAllowedEb
-__ZN6webrtc20pixelBufferFromFrameERKNS_10VideoFrameE
+__ZN6webrtc20pixelBufferFromFrameERKNS_10VideoFrameERKNSt3__18functionIFP10__CVBuffermmEEE
__ZN6webrtc18pixelBufferToFrameEP10__CVBuffer
__ZN3rtc24BasicPacketSocketFactory19CreateAsyncResolverEv
__ZN3rtc24BasicPacketSocketFactoryC2Ev
Modified: trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp (236422 => 236423)
--- trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp 2018-09-24 20:12:57 UTC (rev 236422)
+++ trunk/Source/ThirdParty/libwebrtc/Configurations/libwebrtc.mac.exp 2018-09-24 20:15:17 UTC (rev 236423)
@@ -105,10 +105,10 @@
__ZN6webrtc27SessionDescriptionInterface16RemoveCandidatesERKNSt3__16vectorIN7cricket9CandidateENS1_9allocatorIS4_EEEE
__ZNK6webrtc21IceCandidateInterface10server_urlEv
__ZN6webrtc20setApplicationStatusEb
-__ZN6webrtc32createVideoToolboxDecoderFactoryEv
-__ZN6webrtc32createVideoToolboxEncoderFactoryEv
+__ZN6webrtc26createWebKitDecoderFactoryEv
+__ZN6webrtc26createWebKitEncoderFactoryEv
__ZN6webrtc29setH264HardwareEncoderAllowedEb
-__ZN6webrtc20pixelBufferFromFrameERKNS_10VideoFrameE
+__ZN6webrtc20pixelBufferFromFrameERKNS_10VideoFrameERKNSt3__18functionIFP10__CVBuffermmEEE
__ZN6webrtc18pixelBufferToFrameEP10__CVBuffer
__ZN3rtc24BasicPacketSocketFactory19CreateAsyncResolverEv
__ZN3rtc24BasicPacketSocketFactoryC2Ev
Modified: trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.h (236422 => 236423)
--- trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.h 2018-09-24 20:12:57 UTC (rev 236422)
+++ trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.h 2018-09-24 20:15:17 UTC (rev 236423)
@@ -38,8 +38,8 @@
class VideoEncoderFactory;
class VideoFrame;
-std::unique_ptr<webrtc::VideoEncoderFactory> createVideoToolboxEncoderFactory();
-std::unique_ptr<webrtc::VideoDecoderFactory> createVideoToolboxDecoderFactory();
+std::unique_ptr<webrtc::VideoEncoderFactory> createWebKitEncoderFactory();
+std::unique_ptr<webrtc::VideoDecoderFactory> createWebKitDecoderFactory();
void setApplicationStatus(bool isActive);
@@ -46,7 +46,7 @@
void setH264HardwareEncoderAllowed(bool);
bool isH264HardwareEncoderAllowed();
-CVPixelBufferRef pixelBufferFromFrame(const VideoFrame&);
+CVPixelBufferRef pixelBufferFromFrame(const VideoFrame&, const std::function<CVPixelBufferRef(size_t, size_t)>&);
rtc::scoped_refptr<webrtc::VideoFrameBuffer> pixelBufferToFrame(CVPixelBufferRef);
}
Modified: trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.mm (236422 => 236423)
--- trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.mm 2018-09-24 20:12:57 UTC (rev 236422)
+++ trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitUtilities.mm 2018-09-24 20:15:17 UTC (rev 236423)
@@ -29,6 +29,7 @@
#import "WebRTC/RTCVideoCodecH264.h"
#include "api/video/video_frame.h"
+#include "third_party/libyuv/include/libyuv/convert_from.h"
#include "webrtc/sdk/objc/Framework/Native/src/objc_frame_buffer.h"
#include "webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrame.h"
#include "webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoFrameBuffer.h"
@@ -35,7 +36,6 @@
#include "webrtc/sdk/objc/Framework/Native/api/video_decoder_factory.h"
#include "webrtc/sdk/objc/Framework/Native/api/video_encoder_factory.h"
-
#if !defined(WEBRTC_IOS)
__attribute__((objc_runtime_name("WK_RTCUIApplicationStatusObserver")))
@interface RTCUIApplicationStatusObserver : NSObject
@@ -95,7 +95,7 @@
[[RTCUIApplicationStatusObserver sharedInstance] setInactive];
}
-std::unique_ptr<webrtc::VideoEncoderFactory> createVideoToolboxEncoderFactory()
+std::unique_ptr<webrtc::VideoEncoderFactory> createWebKitEncoderFactory()
{
#if ENABLE_VCP_ENCODER
static std::once_flag onceFlag;
@@ -106,7 +106,7 @@
return ObjCToNativeVideoEncoderFactory([[RTCVideoEncoderFactoryH264 alloc] init]);
}
-std::unique_ptr<webrtc::VideoDecoderFactory> createVideoToolboxDecoderFactory()
+std::unique_ptr<webrtc::VideoDecoderFactory> createWebKitDecoderFactory()
{
return ObjCToNativeVideoDecoderFactory([[RTCVideoDecoderFactoryH264 alloc] init]);
}
@@ -128,11 +128,48 @@
return new rtc::RefCountedObject<ObjCFrameBuffer>(frameBuffer);
}
-CVPixelBufferRef pixelBufferFromFrame(const VideoFrame& frame)
+static bool CopyVideoFrameToPixelBuffer(const rtc::scoped_refptr<webrtc::I420BufferInterface>& frame, CVPixelBufferRef pixel_buffer) {
+ RTC_DCHECK(pixel_buffer);
+ RTC_DCHECK(CVPixelBufferGetPixelFormatType(pixel_buffer) == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange || CVPixelBufferGetPixelFormatType(pixel_buffer) == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange);
+ RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0), static_cast<size_t>(frame->height()));
+ RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0), static_cast<size_t>(frame->width()));
+
+ if (CVPixelBufferLockBaseAddress(pixel_buffer, 0) != kCVReturnSuccess)
+ return false;
+
+ uint8_t* dst_y = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
+ int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
+
+ uint8_t* dst_uv = reinterpret_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
+ int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
+
+ int result = libyuv::I420ToNV12(
+ frame->DataY(), frame->StrideY(),
+ frame->DataU(), frame->StrideU(),
+ frame->DataV(), frame->StrideV(),
+ dst_y, dst_stride_y, dst_uv, dst_stride_uv,
+ frame->width(), frame->height());
+
+ CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
+
+ if (result)
+ return false;
+
+ return true;
+}
+
+
+CVPixelBufferRef pixelBufferFromFrame(const VideoFrame& frame, const std::function<CVPixelBufferRef(size_t, size_t)>& makePixelBuffer)
{
- if (frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative)
- return nullptr;
+ if (frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative) {
+ rtc::scoped_refptr<const I420BufferInterface> buffer = frame.video_frame_buffer()->GetI420();
+ auto pixelBuffer = makePixelBuffer(buffer->width(), buffer->height());
+ if (pixelBuffer)
+ CopyVideoFrameToPixelBuffer(frame.video_frame_buffer()->GetI420(), pixelBuffer);
+ return pixelBuffer;
+ }
+
auto *frameBuffer = static_cast<ObjCFrameBuffer*>(frame.video_frame_buffer().get())->wrapped_frame_buffer();
if (![frameBuffer isKindOfClass:[RTCCVPixelBuffer class]])
return nullptr;
Modified: trunk/Source/WebCore/ChangeLog (236422 => 236423)
--- trunk/Source/WebCore/ChangeLog 2018-09-24 20:12:57 UTC (rev 236422)
+++ trunk/Source/WebCore/ChangeLog 2018-09-24 20:15:17 UTC (rev 236423)
@@ -1,3 +1,27 @@
+2018-09-24 Youenn Fablet <[email protected]>
+
+ Enable conversion of libwebrtc internal frames as CVPixelBuffer
+ https://bugs.webkit.org/show_bug.cgi?id=189892
+
+ Reviewed by Eric Carlson.
+
+ Make sure to handle the case of libwebrtc frames that are not backed by CVPixelBuffer.
+ No observable change of behavior.
+
+ * platform/mediastream/libwebrtc/LibWebRTCProviderCocoa.cpp:
+ (WebCore::LibWebRTCProviderCocoa::createDecoderFactory):
+ (WebCore::LibWebRTCProviderCocoa::createEncoderFactory):
+ Update according renamed methods.
+ * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h:
+ * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm:
+ (WebCore::RealtimeIncomingVideoSourceCocoa::pixelBufferFromVideoFrame):
+ In case of libwebrtc frame that are not backed by CVPixelBuffer, we create
+ a CVPixelBuffer from a pixel buffer pool.
+ This CVPixelBuffer is then filled as part of webrtc::pixelBufferFromFrame.
+ * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm:
+ (WebCore::RealtimeOutgoingVideoSourceCocoa::convertToYUV):
+ Make sure to use preferred pixel buffer format.
+
2018-09-24 Eric Carlson <[email protected]>
[MediaStream] Add mock window capture source
Modified: trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCProviderCocoa.cpp (236422 => 236423)
--- trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCProviderCocoa.cpp 2018-09-24 20:12:57 UTC (rev 236422)
+++ trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCProviderCocoa.cpp 2018-09-24 20:15:17 UTC (rev 236423)
@@ -57,12 +57,12 @@
std::unique_ptr<webrtc::VideoDecoderFactory> LibWebRTCProviderCocoa::createDecoderFactory()
{
- return webrtc::createVideoToolboxDecoderFactory();
+ return webrtc::createWebKitDecoderFactory();
}
std::unique_ptr<webrtc::VideoEncoderFactory> LibWebRTCProviderCocoa::createEncoderFactory()
{
- return webrtc::createVideoToolboxEncoderFactory();
+ return webrtc::createWebKitEncoderFactory();
}
void LibWebRTCProviderCocoa::setActive(bool value)
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h (236422 => 236423)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h 2018-09-24 20:12:57 UTC (rev 236422)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h 2018-09-24 20:15:17 UTC (rev 236423)
@@ -43,7 +43,8 @@
private:
RealtimeIncomingVideoSourceCocoa(rtc::scoped_refptr<webrtc::VideoTrackInterface>&&, String&&);
void processNewSample(CMSampleBufferRef, unsigned, unsigned, MediaSample::VideoRotation);
- CVPixelBufferRef pixelBufferFromVideoFrame(const webrtc::VideoFrame&);
+ RetainPtr<CVPixelBufferRef> pixelBufferFromVideoFrame(const webrtc::VideoFrame&);
+ CVPixelBufferPoolRef pixelBufferPool(size_t width, size_t height);
// rtc::VideoSinkInterface
void OnFrame(const webrtc::VideoFrame&) final;
@@ -55,6 +56,9 @@
#if !RELEASE_LOG_DISABLED
size_t m_numberOfFrames { 0 };
#endif
+ RetainPtr<CVPixelBufferPoolRef> m_pixelBufferPool;
+ size_t m_pixelBufferPoolWidth { 0 };
+ size_t m_pixelBufferPoolHeight { 0 };
};
RetainPtr<CVPixelBufferRef> createBlackPixelBuffer(size_t width, size_t height);
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm (236422 => 236423)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm 2018-09-24 20:12:57 UTC (rev 236422)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm 2018-09-24 20:15:17 UTC (rev 236423)
@@ -91,8 +91,39 @@
return adoptCF(pixelBuffer);
}
-CVPixelBufferRef RealtimeIncomingVideoSourceCocoa::pixelBufferFromVideoFrame(const webrtc::VideoFrame& frame)
+CVPixelBufferPoolRef RealtimeIncomingVideoSourceCocoa::pixelBufferPool(size_t width, size_t height)
{
+ if (!m_pixelBufferPool || m_pixelBufferPoolWidth != width || m_pixelBufferPoolHeight != height) {
+ const OSType videoCaptureFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
+ auto pixelAttributes = @{
+ (__bridge NSString *)kCVPixelBufferWidthKey: @(width),
+ (__bridge NSString *)kCVPixelBufferHeightKey: @(height),
+ (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(videoCaptureFormat),
+ (__bridge NSString *)kCVPixelBufferCGImageCompatibilityKey: @(NO),
+#if PLATFORM(IOS)
+ (__bridge NSString *)kCVPixelFormatOpenGLESCompatibility : @(YES),
+#else
+ (__bridge NSString *)kCVPixelBufferOpenGLCompatibilityKey : @(YES),
+#endif
+ (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{ }
+ };
+
+ CVPixelBufferPoolRef pool = nullptr;
+ auto status = CVPixelBufferPoolCreate(kCFAllocatorDefault, nullptr, (__bridge CFDictionaryRef)pixelAttributes, &pool);
+
+ if (status != kCVReturnSuccess) {
+ RELEASE_LOG(MediaStream, "RealtimeIncomingVideoSourceCocoa::pixelBufferFromVideoFrame failed creating a pixel buffer pool with error %d", status);
+ return nullptr;
+ }
+ m_pixelBufferPool = adoptCF(pool);
+ m_pixelBufferPoolWidth = width;
+ m_pixelBufferPoolHeight = height;
+ }
+ return m_pixelBufferPool.get();
+}
+
+RetainPtr<CVPixelBufferRef> RealtimeIncomingVideoSourceCocoa::pixelBufferFromVideoFrame(const webrtc::VideoFrame& frame)
+{
if (muted()) {
if (!m_blackFrame || m_blackFrameWidth != frame.width() || m_blackFrameHeight != frame.height()) {
m_blackFrameWidth = frame.width();
@@ -101,8 +132,23 @@
}
return m_blackFrame.get();
}
- ASSERT(frame.video_frame_buffer()->type() == webrtc::VideoFrameBuffer::Type::kNative);
- return webrtc::pixelBufferFromFrame(frame);
+
+ RetainPtr<CVPixelBufferRef> newPixelBuffer;
+ return webrtc::pixelBufferFromFrame(frame, [this, &newPixelBuffer](size_t width, size_t height) -> CVPixelBufferRef {
+ auto pixelBufferPool = this->pixelBufferPool(width, height);
+ if (!pixelBufferPool)
+ return nullptr;
+
+ CVPixelBufferRef pixelBuffer = nullptr;
+ auto status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, m_pixelBufferPool.get(), &pixelBuffer);
+
+ if (status != kCVReturnSuccess) {
+ RELEASE_LOG(MediaStream, "RealtimeIncomingVideoSourceCocoa::pixelBufferFromVideoFrame failed creating a pixel buffer with error %d", status);
+ return nullptr;
+ }
+ newPixelBuffer = adoptCF(pixelBuffer);
+ return newPixelBuffer.get();
+ });
}
void RealtimeIncomingVideoSourceCocoa::OnFrame(const webrtc::VideoFrame& frame)
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm (236422 => 236423)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm 2018-09-24 20:12:57 UTC (rev 236422)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm 2018-09-24 20:15:17 UTC (rev 236423)
@@ -31,6 +31,7 @@
#import "Logging.h"
#import "MediaSample.h"
#import "PixelBufferConformerCV.h"
+#import "RealtimeVideoUtilities.h"
#import <pal/cf/CoreMediaSoftLink.h>
#import "CoreVideoSoftLink.h"
#import "VideoToolboxSoftLink.h"
@@ -43,7 +44,7 @@
return nullptr;
if (!m_pixelBufferConformer)
- m_pixelBufferConformer = std::make_unique<PixelBufferConformerCV>((__bridge CFDictionaryRef)@{ (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_420YpCbCr8Planar) });
+ m_pixelBufferConformer = std::make_unique<PixelBufferConformerCV>((__bridge CFDictionaryRef)@{ (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(preferedPixelBufferFormat()) });
return m_pixelBufferConformer->convert(pixelBuffer);
}
