Modified: trunk/Source/ThirdParty/libwebrtc/ChangeLog (278271 => 278272)
--- trunk/Source/ThirdParty/libwebrtc/ChangeLog 2021-05-31 06:39:50 UTC (rev 278271)
+++ trunk/Source/ThirdParty/libwebrtc/ChangeLog 2021-05-31 07:28:32 UTC (rev 278272)
@@ -1,3 +1,33 @@
+2021-05-31 Youenn Fablet <[email protected]>
+
+ Use tighter bitrate allocation rules for WebRTC H264 software encoder
+ https://bugs.webkit.org/show_bug.cgi?id=226319
+ <rdar://73150695>
+
+ Reviewed by Eric Carlson.
+
+ Software H264 encoder is sometimes overshooting target bitrate in which case WebRTC backend will start dropping frames.
+ The encoder might then think it is on target and will not try to increase compression.
+ This makes it possible to be locked in a very low frame rate but high quality image situation.
+ It is often better to preserve frame rate and lower quality, the application could always lower frame rate if desired.
+
+ To do so, we detect whether the encoder is using software code path or not.
+ If so, we compute the actual frame rate and compare it with the expected frame rate.
+ If the actual frame rate is twice smaller or even below, we enter in a low frame rate mode.
+ Otherwise, we are in a regular frame rate mode where we apply the normal bitrate rules.
+ In the low frame rate mode, we divide the target bitrate by 3 so as to quickly recover frame rate.
+
+ This works well in situations where motion increases from time to time.
+ It is still not perfect for instance in case the video is muted and gets unmuted or when the scene is completely still and suddenly large motion happens.
+ In those cases, frame rate is recovered after a minute or so according my testing.
+
+ * Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm:
+ (-[RTCVideoEncoderH264 initWithCodecInfo:]):
+ (-[RTCVideoEncoderH264 encode:codecSpecificInfo:frameTypes:]):
+ (-[RTCVideoEncoderH264 resetCompressionSessionWithPixelFormat:]):
+ (-[RTCVideoEncoderH264 setEncoderBitrateBps:frameRate:]):
+ (-[RTCVideoEncoderH264 updateBitRateAccordingActualFrameRate]):
+
2021-05-20 Youenn Fablet <[email protected]>
Enable VCP for H264 baseline
Modified: trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm (278271 => 278272)
--- trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm 2021-05-31 06:39:50 UTC (rev 278271)
+++ trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm 2021-05-31 07:28:32 UTC (rev 278272)
@@ -35,6 +35,7 @@
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/buffer.h"
#include "rtc_base/logging.h"
+#include "rtc_base/system/arch.h"
#include "rtc_base/time_utils.h"
#include "sdk/objc/components/video_codec/nalu_rewriter.h"
#include "third_party/libyuv/include/libyuv/convert_from.h"
@@ -60,7 +61,7 @@
timestamp:(uint32_t)timestamp
rotation:(RTCVideoRotation)rotation
isKeyFrameRequired:(bool)isKeyFrameRequired;
-
+- (void)updateBitRateAccordingActualFrameRate;
@end
namespace { // anonymous namespace
@@ -76,6 +77,9 @@
const OSType kNV12PixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
+const int kBelowFrameRateThreshold = 2;
+const int kBelowFrameRateBitRateDecrease = 3;
+
// Struct that we pass to the encoder per frame to encode. We receive it again
// in the encoder callback.
struct RTCFrameEncodeParams {
@@ -349,6 +353,10 @@
bool _disableEncoding;
bool _isKeyFrameRequired;
bool _isH264LowLatencyEncoderEnabled;
+ bool _isUsingSoftwareEncoder;
+ bool _isBelowExpectedFrameRate;
+ uint32_t _frameCount;
+ int64_t _lastFrameRateEstimationTime;
}
// .5 is set as a mininum to prevent overcompensating for large temporary
@@ -377,7 +385,10 @@
}
_isKeyFrameRequired = false;
_isH264LowLatencyEncoderEnabled = true;
-
+ _isUsingSoftwareEncoder = false;
+ _isBelowExpectedFrameRate = false;
+ _frameCount = 0;
+ _lastFrameRateEstimationTime = 0;
return self;
}
@@ -442,6 +453,9 @@
if (_disableEncoding) {
return WEBRTC_VIDEO_CODEC_ERROR;
}
+ if (_isUsingSoftwareEncoder) {
+ [self updateBitRateAccordingActualFrameRate];
+ }
CVPixelBufferRef pixelBuffer = nullptr;
if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
@@ -751,6 +765,12 @@
nullptr,
&_vcpCompressionSession);
}
+#elif defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) && !(ENABLE_VCP_FOR_H264_BASELINE) && defined(WEBRTC_ARCH_X86_FAMILY)
+ CFBooleanRef hwaccl_enabled = nullptr;
+ if (status == noErr) {
+ auto result = VTSessionCopyProperty(_vtCompressionSession, kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, nullptr, &hwaccl_enabled);
+ _isUsingSoftwareEncoder = result == noErr ? !CFBooleanGetValue(hwaccl_enabled) : _width <= 480 && _height <= 480;
+ }
#else
// Provided encoder should be good enough.
#endif // defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) && ENABLE_VCP_ENCODER
@@ -839,7 +859,8 @@
- (void)setEncoderBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate {
if ([self hasCompressionSession]) {
- SetVTSessionProperty(_vtCompressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps);
+ auto actualTarget = _isBelowExpectedFrameRate ? bitrateBps / kBelowFrameRateBitRateDecrease : bitrateBps;
+ SetVTSessionProperty(_vtCompressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_AverageBitRate, actualTarget);
// With zero |_maxAllowedFrameRate|, we fall back to automatic frame rate detection.
if (_maxAllowedFrameRate > 0) {
@@ -849,7 +870,7 @@
// TODO(tkchin): Add a helper method to set array value.
int64_t dataLimitBytesPerSecondValue =
- static_cast<int64_t>(bitrateBps * kLimitToAverageBitRateFactor / 8);
+ static_cast<int64_t>(_isBelowExpectedFrameRate ? actualTarget / 8 : actualTarget * kLimitToAverageBitRateFactor / 8);
CFNumberRef bytesPerSecond =
CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitBytesPerSecondValue);
int64_t _oneSecondValue_ = 1;
@@ -875,6 +896,34 @@
}
}
+- (void)updateBitRateAccordingActualFrameRate {
+ _frameCount++;
+ auto currentTime = rtc::TimeMillis();
+ if (!_lastFrameRateEstimationTime) {
+ _lastFrameRateEstimationTime = currentTime;
+ return;
+ }
+
+ auto timeDifference = currentTime - _lastFrameRateEstimationTime;
+
+ // Let's not check too often.
+ if (timeDifference < 1000) {
+ return;
+ }
+
+ auto frameRate = _frameCount * 1000 / timeDifference;
+ _lastFrameRateEstimationTime = currentTime;
+ _frameCount = 0;
+
+ bool isBelow = frameRate * kBelowFrameRateThreshold < _encoderFrameRate;
+ if (isBelow == _isBelowExpectedFrameRate) {
+ return;
+ }
+
+ _isBelowExpectedFrameRate = isBelow;
+ [self setEncoderBitrateBps:_encoderBitrateBps frameRate:_encoderFrameRate];
+}
+
- (void)frameWasEncoded:(OSStatus)status
flags:(VTEncodeInfoFlags)infoFlags
sampleBuffer:(CMSampleBufferRef)sampleBuffer
