Modified: trunk/Source/WebCore/ChangeLog (235809 => 235810)
--- trunk/Source/WebCore/ChangeLog 2018-09-07 21:57:47 UTC (rev 235809)
+++ trunk/Source/WebCore/ChangeLog 2018-09-07 22:28:18 UTC (rev 235810)
@@ -1,5 +1,26 @@
2018-09-07 Youenn Fablet <[email protected]>
+ RealtimeOutgoingVideoSourceCocoa should use VTImageRotationSession to rotate CVPixelBuffers
+ https://bugs.webkit.org/show_bug.cgi?id=189427
+
+ Reviewed by Eric Carlson.
+
+ Previously, we were relying on libwebrtc utils to do the rotation.
+ This is inefficient compared to VTImageRotateSession and also induces additional memory cost
+ since libwebrtc is rotating using its own buffers and the encoder will convert this buffer back to a CVPixelBuffer.
+
+ Instead use VTImageRotationSession when rotation must be done at sending side.
+ Covered by webrtc/video-rotation.html.
+
+ * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp:
+ (WebCore::RealtimeOutgoingVideoSourceCocoa::sampleBufferUpdated):
+ * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h:
+ * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm:
+ (WebCore::computeRotatedWidthAndHeight):
+ (WebCore::RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer):
+
+2018-09-07 Youenn Fablet <[email protected]>
+
Add support for unified plan transceivers
https://bugs.webkit.org/show_bug.cgi?id=189390
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp (235809 => 235810)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp 2018-09-07 21:57:47 UTC (rev 235809)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp 2018-09-07 22:28:18 UTC (rev 235810)
@@ -152,23 +152,14 @@
auto pixelBuffer = static_cast<CVPixelBufferRef>(CMSampleBufferGetImageBuffer(sample.platformSample().sample.cmSampleBuffer));
auto pixelFormatType = CVPixelBufferGetPixelFormatType(pixelBuffer);
- rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer;
- RetainPtr<CVPixelBufferRef> convertedBuffer;
- if (pixelFormatType == kCVPixelFormatType_420YpCbCr8Planar || pixelFormatType == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
- buffer = webrtc::pixelBufferToFrame(pixelBuffer);
- else {
+ RetainPtr<CVPixelBufferRef> convertedBuffer = pixelBuffer;
+ if (pixelFormatType != kCVPixelFormatType_420YpCbCr8Planar && pixelFormatType != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
convertedBuffer = convertToYUV(pixelBuffer);
- buffer = webrtc::pixelBufferToFrame(convertedBuffer.get());
- }
- if (m_shouldApplyRotation && m_currentRotation != webrtc::kVideoRotation_0) {
- // FIXME: We should make AVVideoCaptureSource handle the rotation whenever possible.
- // This implementation is inefficient, we should rotate on the CMSampleBuffer directly instead of doing this double allocation.
- auto rotatedBuffer = buffer->ToI420();
- ASSERT(rotatedBuffer);
- buffer = webrtc::I420Buffer::Rotate(*rotatedBuffer, m_currentRotation);
- }
- sendFrame(WTFMove(buffer));
+ if (m_shouldApplyRotation && m_currentRotation != webrtc::kVideoRotation_0)
+ convertedBuffer = rotatePixelBuffer(convertedBuffer.get(), m_currentRotation);
+
+ sendFrame(webrtc::pixelBufferToFrame(convertedBuffer.get()));
}
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h (235809 => 235810)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h 2018-09-07 21:57:47 UTC (rev 235809)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h 2018-09-07 22:28:18 UTC (rev 235810)
@@ -29,7 +29,11 @@
#include "RealtimeOutgoingVideoSource.h"
#include "PixelBufferConformerCV.h"
+#include <webrtc/api/video/video_rotation.h>
+typedef struct OpaqueVTImageRotationSession* VTImageRotationSessionRef;
+typedef struct __CVPixelBufferPool* CVPixelBufferPoolRef;
+
namespace WebCore {
class RealtimeOutgoingVideoSourceCocoa final : public RealtimeOutgoingVideoSource {
@@ -43,8 +47,15 @@
void sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample&) final;
RetainPtr<CVPixelBufferRef> convertToYUV(CVPixelBufferRef);
+ RetainPtr<CVPixelBufferRef> rotatePixelBuffer(CVPixelBufferRef, webrtc::VideoRotation);
std::unique_ptr<PixelBufferConformerCV> m_pixelBufferConformer;
+ RetainPtr<VTImageRotationSessionRef> m_rotationSession;
+ RetainPtr<CVPixelBufferPoolRef> m_rotationPool;
+ webrtc::VideoRotation m_currentRotationAngle { webrtc::kVideoRotation_0 };
+ size_t m_rotatedWidth { 0 };
+ size_t m_rotatedHeight { 0 };
+ OSType m_rotatedFormat;
#if !RELEASE_LOG_DISABLED
size_t m_numberOfFrames { 0 };
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm (235809 => 235810)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm 2018-09-07 21:57:47 UTC (rev 235809)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.mm 2018-09-07 22:28:18 UTC (rev 235810)
@@ -28,9 +28,12 @@
#if USE(LIBWEBRTC)
+#import "Logging.h"
+#import "MediaSample.h"
#import "PixelBufferConformerCV.h"
#import <pal/cf/CoreMediaSoftLink.h>
#import "CoreVideoSoftLink.h"
+#import "VideoToolboxSoftLink.h"
namespace WebCore {
@@ -45,6 +48,85 @@
return m_pixelBufferConformer->convert(pixelBuffer);
}
+static inline void computeRotatedWidthAndHeight(CVPixelBufferRef pixelBuffer, webrtc::VideoRotation rotation, size_t& width, size_t& height)
+{
+ switch (rotation) {
+ case webrtc::kVideoRotation_0:
+ case webrtc::kVideoRotation_180:
+ width = CVPixelBufferGetWidth(pixelBuffer);
+ height = CVPixelBufferGetHeight(pixelBuffer);
+ return;
+ case webrtc::kVideoRotation_90:
+ case webrtc::kVideoRotation_270:
+ width = CVPixelBufferGetHeight(pixelBuffer);
+ height = CVPixelBufferGetWidth(pixelBuffer);
+ return;
+ }
+}
+
+RetainPtr<CVPixelBufferRef> RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer(CVPixelBufferRef pixelBuffer, webrtc::VideoRotation rotation)
+{
+ ASSERT(rotation);
+ if (!rotation)
+ return pixelBuffer;
+
+ if (!m_rotationSession || rotation != m_currentRotation) {
+ VTImageRotationSessionRef rawRotationSession = nullptr;
+ auto status = VTImageRotationSessionCreate(kCFAllocatorDefault, rotation, &rawRotationSession);
+ if (status != noErr) {
+ RELEASE_LOG(MediaStream, "RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer failed creating a rotation session with error %d", status);
+ return nullptr;
+ }
+
+ m_rotationSession = adoptCF(rawRotationSession);
+ m_currentRotation = rotation;
+
+ VTImageRotationSessionSetProperty(rawRotationSession, kVTImageRotationPropertyKey_EnableHighSpeedTransfer, kCFBooleanTrue);
+ }
+
+ size_t rotatedWidth, rotatedHeight;
+ computeRotatedWidthAndHeight(pixelBuffer, rotation, rotatedWidth, rotatedHeight);
+ auto format = CVPixelBufferGetPixelFormatType(pixelBuffer);
+ if (!m_rotationPool || rotatedWidth != m_rotatedWidth || rotatedHeight != m_rotatedHeight || format != m_rotatedFormat) {
+ auto pixelAttributes = @{
+ (__bridge NSString *)kCVPixelBufferWidthKey: @(rotatedWidth),
+ (__bridge NSString *)kCVPixelBufferHeightKey: @(rotatedHeight),
+ (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(format),
+ (__bridge NSString *)kCVPixelBufferCGImageCompatibilityKey: @NO,
+ };
+
+ CVPixelBufferPoolRef pool = nullptr;
+ auto status = CVPixelBufferPoolCreate(kCFAllocatorDefault, nullptr, (__bridge CFDictionaryRef)pixelAttributes, &pool);
+
+ if (status != kCVReturnSuccess) {
+ RELEASE_LOG(MediaStream, "RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer failed creating a pixel buffer pool with error %d", status);
+ return nullptr;
+ }
+ m_rotationPool = adoptCF(pool);
+
+ m_rotatedWidth = rotatedWidth;
+ m_rotatedHeight = rotatedHeight;
+ m_rotatedFormat = format;
+ }
+
+ CVPixelBufferRef rawRotatedBuffer = nullptr;
+ auto status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, m_rotationPool.get(), &rawRotatedBuffer);
+
+ if (status != kCVReturnSuccess) {
+ RELEASE_LOG(MediaStream, "RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer failed creating a pixel buffer with error %d", status);
+ return nullptr;
+ }
+ RetainPtr<CVPixelBufferRef> rotatedBuffer = adoptCF(rawRotatedBuffer);
+
+ status = VTImageRotationSessionTransferImage(m_rotationSession.get(), pixelBuffer, rotatedBuffer.get());
+
+ if (status != noErr) {
+ RELEASE_LOG(MediaStream, "RealtimeOutgoingVideoSourceCocoa::rotatePixelBuffer failed rotating with error %d", status);
+ return nullptr;
+ }
+ return rotatedBuffer;
+}
+
} // namespace WebCore
#endif // USE(LIBWEBRTC)
