Diff
Modified: branches/safari-604-branch/Source/WebCore/ChangeLog (223304 => 223305)
--- branches/safari-604-branch/Source/WebCore/ChangeLog 2017-10-13 22:26:06 UTC (rev 223304)
+++ branches/safari-604-branch/Source/WebCore/ChangeLog 2017-10-13 22:26:09 UTC (rev 223305)
@@ -2,6 +2,72 @@
Apply patch. rdar://problem/34891297
+ Cherry-pick of r222803 <rdar://problem/34745579>
+
+ 2017-10-03 Jer Noble <jer.no...@apple.com>
+
+ Implement quality-of-service tiers in WebCoreDecompressionSession
+ https://bugs.webkit.org/show_bug.cgi?id=177769
+
+ Reviewed by Dean Jackson.
+
+ VTDecompressionSession will suggest quality-of-service tiers to be used when decompression
+ can't keep up with playback speed. Use a simple exponential-moving-average heuristic to
+ determine when to move up and down the tiers.
+
+ Drive-by fix: When frames are so late that they miss the display deadline, mark them as
+ dropped rather than just delayed.
+
+ * platform/graphics/cocoa/WebCoreDecompressionSession.h:
+ * platform/graphics/cocoa/WebCoreDecompressionSession.mm:
+ (WebCore::WebCoreDecompressionSession::ensureDecompressionSessionForSample):
+ (WebCore::WebCoreDecompressionSession::decodeSample):
+ (WebCore::WebCoreDecompressionSession::handleDecompressionOutput):
+ (WebCore::WebCoreDecompressionSession::automaticDequeue):
+ (WebCore::WebCoreDecompressionSession::enqueueDecodedSample):
+ (WebCore::WebCoreDecompressionSession::resetQosTier):
+ (WebCore::WebCoreDecompressionSession::increaseQosTier):
+ (WebCore::WebCoreDecompressionSession::decreaseQosTier):
+ (WebCore::WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics):
+ * platform/cocoa/VideoToolboxSoftLink.cpp:
+ * platform/cocoa/VideoToolboxSoftLink.h:
+
+ 2017-10-12 Jer Noble <jer.no...@apple.com>
+
+ Cherry-pick of r222803 <rdar://problem/34745579>
+
+ 2017-10-03 Jer Noble <jer.no...@apple.com>
+
+ Implement quality-of-service tiers in WebCoreDecompressionSession
+ https://bugs.webkit.org/show_bug.cgi?id=177769
+
+ Reviewed by Dean Jackson.
+
+ VTDecompressionSession will suggest quality-of-service tiers to be used when decompression
+ can't keep up with playback speed. Use a simple exponential-moving-average heuristic to
+ determine when to move up and down the tiers.
+
+ Drive-by fix: When frames are so late that they miss the display deadline, mark them as
+ dropped rather than just delayed.
+
+ * platform/graphics/cocoa/WebCoreDecompressionSession.h:
+ * platform/graphics/cocoa/WebCoreDecompressionSession.mm:
+ (WebCore::WebCoreDecompressionSession::ensureDecompressionSessionForSample):
+ (WebCore::WebCoreDecompressionSession::decodeSample):
+ (WebCore::WebCoreDecompressionSession::handleDecompressionOutput):
+ (WebCore::WebCoreDecompressionSession::automaticDequeue):
+ (WebCore::WebCoreDecompressionSession::enqueueDecodedSample):
+ (WebCore::WebCoreDecompressionSession::resetQosTier):
+ (WebCore::WebCoreDecompressionSession::increaseQosTier):
+ (WebCore::WebCoreDecompressionSession::decreaseQosTier):
+ (WebCore::WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics):
+ * platform/cocoa/VideoToolboxSoftLink.cpp:
+ * platform/cocoa/VideoToolboxSoftLink.h:
+
+2017-10-12 Jason Marcell <jmarc...@apple.com>
+
+ Apply patch. rdar://problem/34891297
+
Partial cherry-pick of r222225 <rdar://problem/34745579>, only the changes to WebCoreDecompressionSession.
2017-09-19 Jer Noble <jer.no...@apple.com>
Modified: branches/safari-604-branch/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.cpp (223304 => 223305)
--- branches/safari-604-branch/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.cpp 2017-10-13 22:26:06 UTC (rev 223304)
+++ branches/safari-604-branch/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.cpp 2017-10-13 22:26:09 UTC (rev 223305)
@@ -33,6 +33,7 @@
SOFT_LINK_FRAMEWORK_FOR_SOURCE(WebCore, VideoToolbox)
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTSessionCopyProperty, OSStatus, (VTSessionRef session, CFStringRef propertyKey, CFAllocatorRef allocator, void* propertyValueOut), (session, propertyKey, allocator, propertyValueOut))
+SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTSessionSetProperties, OSStatus, (VTSessionRef session, CFDictionaryRef propertyDictionary), (session, propertyDictionary))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTDecompressionSessionCreate, OSStatus, (CFAllocatorRef allocator, CMVideoFormatDescriptionRef videoFormatDescription, CFDictionaryRef videoDecoderSpecification, CFDictionaryRef destinationImageBufferAttributes, const VTDecompressionOutputCallbackRecord* outputCallback, VTDecompressionSessionRef* decompressionSessionOut), (allocator, videoFormatDescription, videoDecoderSpecification, destinationImageBufferAttributes, outputCallback, decompressionSessionOut))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTDecompressionSessionCanAcceptFormatDescription, Boolean, (VTDecompressionSessionRef session, CMFormatDescriptionRef newFormatDesc), (session, newFormatDesc))
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, VideoToolbox, VTDecompressionSessionWaitForAsynchronousFrames, OSStatus, (VTDecompressionSessionRef session), (session))
@@ -44,6 +45,7 @@
SOFT_LINK_FUNCTION_MAY_FAIL_FOR_SOURCE(WebCore, VideoToolbox, VTCreateCGImageFromCVPixelBuffer, OSStatus, (CVPixelBufferRef pixelBuffer, CFDictionaryRef options, CGImageRef* imageOut), (pixelBuffer, options, imageOut))
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTDecompressionPropertyKey_PixelBufferPool, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTImageRotationPropertyKey_EnableHighSpeedTransfer, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTImageRotationPropertyKey_FlipHorizontalOrientation, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, VideoToolbox, kVTImageRotationPropertyKey_FlipVerticalOrientation, CFStringRef)
Modified: branches/safari-604-branch/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.h (223304 => 223305)
--- branches/safari-604-branch/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.h 2017-10-13 22:26:06 UTC (rev 223304)
+++ branches/safari-604-branch/Source/WebCore/platform/cocoa/VideoToolboxSoftLink.h 2017-10-13 22:26:09 UTC (rev 223305)
@@ -34,6 +34,8 @@
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTSessionCopyProperty, OSStatus, (VTSessionRef session, CFStringRef propertyKey, CFAllocatorRef allocator, void* propertyValueOut), (session, propertyKey, allocator, propertyValueOut))
#define VTSessionCopyProperty softLink_VideoToolbox_VTSessionCopyProperty
+SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTSessionSetProperties, OSStatus, (VTSessionRef session, CFDictionaryRef propertyDictionary), (session, propertyDictionary))
+#define VTSessionSetProperties softLink_VideoToolbox_VTSessionSetProperties
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTDecompressionSessionCreate, OSStatus, (CFAllocatorRef allocator, CMVideoFormatDescriptionRef videoFormatDescription, CFDictionaryRef videoDecoderSpecification, CFDictionaryRef destinationImageBufferAttributes, const VTDecompressionOutputCallbackRecord* outputCallback, VTDecompressionSessionRef* decompressionSessionOut), (allocator, videoFormatDescription, videoDecoderSpecification, destinationImageBufferAttributes, outputCallback, decompressionSessionOut))
#define VTDecompressionSessionCreate softLink_VideoToolbox_VTDecompressionSessionCreate
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, VideoToolbox, VTDecompressionSessionCanAcceptFormatDescription, Boolean, (VTDecompressionSessionRef session, CMFormatDescriptionRef newFormatDesc), (session, newFormatDesc))
@@ -58,6 +60,8 @@
#define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder get_VideoToolbox_kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder()
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTDecompressionPropertyKey_PixelBufferPool, CFStringRef)
#define kVTDecompressionPropertyKey_PixelBufferPool get_VideoToolbox_kVTDecompressionPropertyKey_PixelBufferPool()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers, CFStringRef)
+#define kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers get_VideoToolbox_kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers()
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTImageRotationPropertyKey_EnableHighSpeedTransfer, CFStringRef)
#define kVTImageRotationPropertyKey_EnableHighSpeedTransfer get_VideoToolbox_kVTImageRotationPropertyKey_EnableHighSpeedTransfer()
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, VideoToolbox, kVTImageRotationPropertyKey_FlipHorizontalOrientation, CFStringRef)
Modified: branches/safari-604-branch/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.h (223304 => 223305)
--- branches/safari-604-branch/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.h 2017-10-13 22:26:06 UTC (rev 223304)
+++ branches/safari-604-branch/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.h 2017-10-13 22:26:09 UTC (rev 223305)
@@ -36,6 +36,7 @@
#include <wtf/ThreadSafeRefCounted.h>
typedef CFTypeRef CMBufferRef;
+typedef const struct __CFArray * CFArrayRef;
typedef struct opaqueCMBufferQueue *CMBufferQueueRef;
typedef struct opaqueCMSampleBuffer *CMSampleBufferRef;
typedef struct OpaqueCMTimebase* CMTimebaseRef;
@@ -99,6 +100,11 @@
static CFComparisonResult compareBuffers(CMBufferRef buf1, CMBufferRef buf2, void* refcon);
void maybeBecomeReadyForMoreMediaData();
+ void resetQosTier();
+ void increaseQosTier();
+ void decreaseQosTier();
+ void updateQosWithDecodeTimeStatistics(double ratio);
+
static const CMItemCount kMaximumCapacity = 120;
static const CMItemCount kHighWaterMark = 60;
static const CMItemCount kLowWaterMark = 15;
@@ -114,6 +120,10 @@
OSObjectPtr<dispatch_source_t> m_timerSource;
std::function<void()> m_notificationCallback;
std::function<void()> m_hasAvailableFrameCallback;
+ RetainPtr<CFArrayRef> m_qosTiers;
+ long m_currentQosTier { 0 };
+ unsigned long m_framesSinceLastQosCheck { 0 };
+ double m_decodeRatioMovingAverage { 0 };
bool m_invalidated { false };
int m_framesBeingDecoded { 0 };
Modified: branches/safari-604-branch/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm (223304 => 223305)
--- branches/safari-604-branch/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm 2017-10-13 22:26:06 UTC (rev 223304)
+++ branches/safari-604-branch/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm 2017-10-13 22:26:09 UTC (rev 223305)
@@ -33,6 +33,8 @@
#import "PixelBufferConformerCV.h"
#import <CoreMedia/CMBufferQueue.h>
#import <CoreMedia/CMFormatDescription.h>
+#import <MediaTimeAVFoundation.h>
+#import <wtf/CurrentTime.h>
#import <wtf/MainThread.h>
#import <wtf/MediaTime.h>
#import <wtf/StringPrintStream.h>
@@ -228,8 +230,14 @@
attributes = @{(NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)};
}
VTDecompressionSessionRef decompressionSessionOut = nullptr;
- if (noErr == VTDecompressionSessionCreate(kCFAllocatorDefault, videoFormatDescription, (CFDictionaryRef)videoDecoderSpecification, (CFDictionaryRef)attributes, nullptr, &decompressionSessionOut))
+ if (noErr == VTDecompressionSessionCreate(kCFAllocatorDefault, videoFormatDescription, (CFDictionaryRef)videoDecoderSpecification, (CFDictionaryRef)attributes, nullptr, &decompressionSessionOut)) {
m_decompressionSession = adoptCF(decompressionSessionOut);
+ CFArrayRef rawSuggestedQualityOfServiceTiers = nullptr;
+ VTSessionCopyProperty(decompressionSessionOut, kVTDecompressionPropertyKey_SuggestedQualityOfServiceTiers, kCFAllocatorDefault, &rawSuggestedQualityOfServiceTiers);
+ m_qosTiers = adoptCF(rawSuggestedQualityOfServiceTiers);
+ m_currentQosTier = 0;
+ resetQosTier();
+ }
}
}
@@ -252,7 +260,11 @@
return;
}
- VTDecompressionSessionDecodeFrameWithOutputHandler(m_decompressionSession.get(), sample, flags, nullptr, [this, displaying] (OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef imageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration) {
+ double startTime = monotonicallyIncreasingTime();
+ VTDecompressionSessionDecodeFrameWithOutputHandler(m_decompressionSession.get(), sample, flags, nullptr, [this, displaying, startTime](OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef imageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration) {
+ double deltaRatio = (monotonicallyIncreasingTime() - startTime) / CMTimeGetSeconds(presentationDuration);
+
+ updateQosWithDecodeTimeStatistics(deltaRatio);
handleDecompressionOutput(displaying, status, infoFlags, imageBuffer, presentationTimeStamp, presentationDuration);
});
}
@@ -302,13 +314,6 @@
return;
}
- if (displaying && m_timebase) {
- auto currentTime = CMTimebaseGetTime(m_timebase.get());
- auto currentRate = CMTimebaseGetRate(m_timebase.get());
- if (currentRate > 0 && CMTimeCompare(presentationTimeStamp, currentTime) < 0)
- m_totalFrameDelay += toMediaTime(CMTimeSubtract(currentTime, presentationTimeStamp));
- }
-
dispatch_async(m_enqueingQueue.get(), [protectedThis = makeRefPtr(this), status, imageSampleBuffer = adoptCF(rawImageSampleBuffer), infoFlags, displaying] {
UNUSED_PARAM(infoFlags);
protectedThis->enqueueDecodedSample(imageSampleBuffer.get(), displaying);
@@ -379,6 +384,27 @@
return;
}
+ bool shouldNotify = true;
+
+ if (displaying && m_timebase) {
+ auto currentRate = CMTimebaseGetRate(m_timebase.get());
+ auto currentTime = toMediaTime(CMTimebaseGetTime(m_timebase.get()));
+ auto presentationStartTime = toMediaTime(CMSampleBufferGetPresentationTimeStamp(sample));
+ auto presentationEndTime = presentationStartTime + toMediaTime(CMSampleBufferGetDuration(sample));
+ if (currentTime < presentationStartTime || currentTime >= presentationEndTime)
+ shouldNotify = false;
+
+ if (currentRate > 0 && presentationEndTime < currentTime) {
+#if !LOG_DISABLED
+ auto begin = toMediaTime(CMBufferQueueGetFirstPresentationTimeStamp(m_producerQueue.get()));
+ auto end = toMediaTime(CMBufferQueueGetEndPresentationTimeStamp(m_producerQueue.get()));
+ LOG(Media, "WebCoreDecompressionSession::enqueueDecodedSample(%p) - dropping frame late by %s, framesBeingDecoded(%d), producerQueue(%s -> %s)", this, toString(presentationEndTime - currentTime).utf8().data(), m_framesBeingDecoded, toString(begin).utf8().data(), toString(end).utf8().data());
+#endif
+ ++m_droppedVideoFrames;
+ return;
+ }
+ }
+
CMBufferQueueEnqueue(m_producerQueue.get(), sample);
#if !LOG_DISABLED
@@ -394,13 +420,8 @@
if (!m_hasAvailableFrameCallback)
return;
- if (m_timebase) {
- auto currentTime = toMediaTime(CMTimebaseGetTime(m_timebase.get()));
- auto presentationStartTime = toMediaTime(CMSampleBufferGetPresentationTimeStamp(sample));
- auto presentationEndTime = presentationStartTime + toMediaTime(CMSampleBufferGetDuration(sample));
- if (currentTime < presentationStartTime || currentTime >= presentationEndTime)
- return;
- }
+ if (!shouldNotify)
+ return;
dispatch_async(dispatch_get_main_queue(), [protectedThis = makeRefPtr(this), callback = WTFMove(m_hasAvailableFrameCallback)] {
callback();
@@ -504,6 +525,9 @@
CMBufferQueueReset(protectedThis->m_producerQueue.get());
dispatch_sync(protectedThis->m_enqueingQueue.get(), [protectedThis] {
CMBufferQueueReset(protectedThis->m_consumerQueue.get());
+ protectedThis->m_framesSinceLastQosCheck = 0;
+ protectedThis->m_currentQosTier = 0;
+ protectedThis->resetQosTier();
});
});
}
@@ -534,6 +558,71 @@
return (CFComparisonResult)CMTimeCompare(getPresentationTime(buf1, refcon), getPresentationTime(buf2, refcon));
}
+void WebCoreDecompressionSession::resetQosTier()
+{
+ if (!m_qosTiers || !m_decompressionSession)
+ return;
+
+ if (m_currentQosTier < 0 || m_currentQosTier >= CFArrayGetCount(m_qosTiers.get()))
+ return;
+
+ auto tier = (CFDictionaryRef)CFArrayGetValueAtIndex(m_qosTiers.get(), m_currentQosTier);
+ LOG(Media, "WebCoreDecompressionSession::resetQosTier(%p) - currentQosTier(%ld), tier(%@)", this, m_currentQosTier, [(NSDictionary *)tier description]);
+
+ VTSessionSetProperties(m_decompressionSession.get(), tier);
+ m_framesSinceLastQosCheck = 0;
}
+void WebCoreDecompressionSession::increaseQosTier()
+{
+ if (!m_qosTiers)
+ return;
+
+ if (m_currentQosTier + 1 >= CFArrayGetCount(m_qosTiers.get()))
+ return;
+
+ ++m_currentQosTier;
+ resetQosTier();
+}
+
+void WebCoreDecompressionSession::decreaseQosTier()
+{
+ if (!m_qosTiers)
+ return;
+
+ if (m_currentQosTier <= 0)
+ return;
+
+ --m_currentQosTier;
+ resetQosTier();
+}
+
+void WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics(double ratio)
+{
+ static const double kMovingAverageAlphaValue = 0.1;
+ static const unsigned kNumberOfFramesBeforeSwitchingTiers = 60;
+ static const double kHighWaterDecodeRatio = 1.;
+ static const double kLowWaterDecodeRatio = 0.5;
+
+ if (!m_timebase)
+ return;
+
+ double rate = CMTimebaseGetRate(m_timebase.get());
+ if (!rate)
+ rate = 1;
+
+ m_decodeRatioMovingAverage += kMovingAverageAlphaValue * (ratio - m_decodeRatioMovingAverage) * rate;
+ if (++m_framesSinceLastQosCheck < kNumberOfFramesBeforeSwitchingTiers)
+ return;
+
+ LOG(Media, "WebCoreDecompressionSession::updateQosWithDecodeTimeStatistics(%p) - framesSinceLastQosCheck(%ld), decodeRatioMovingAverage(%g)", this, m_framesSinceLastQosCheck, m_decodeRatioMovingAverage);
+ if (m_decodeRatioMovingAverage > kHighWaterDecodeRatio)
+ increaseQosTier();
+ else if (m_decodeRatioMovingAverage < kLowWaterDecodeRatio)
+ decreaseQosTier();
+ m_framesSinceLastQosCheck = 0;
+}
+
+}
+
#endif
