- Revision
- 208444
- Author
- [email protected]
- Date
- 2016-11-09 10:27:37 -0800 (Wed, 09 Nov 2016)
Log Message
[MediaStream][Mac] Mark captured video frames as ready for display immediately
https://bugs.webkit.org/show_bug.cgi?id=164482
<rdar://problem/29139073>
Reviewed by Jer Noble.
* platform/cf/CoreMediaSoftLink.cpp: Add new constant.
* platform/cf/CoreMediaSoftLink.h:
* platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
* platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::requestNotificationWhenReadyForMediaData):
New, ask register for a callback when the sample buffer display layer is ready
for more media data.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBuffer): Don't change
the sample timestamps, assume the caller has configured the sample correctly.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::prepareVideoSampleBufferFromTrack): Don't
drop frames when the display layer isn't ready.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::destroyLayer): Call stopRequestingMediaData.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::sampleBufferUpdated):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack): Deleted.
* platform/mediastream/mac/AVVideoCaptureSource.mm:
(WebCore::AVVideoCaptureSource::setupCaptureSession): Tell the video output to always discard
late video frames, we don't need them.
(WebCore::AVVideoCaptureSource::processNewFrame): Add a kCMSampleAttachmentKey_DisplayImmediately
attachment.
Modified Paths
Diff
Modified: trunk/Source/WebCore/ChangeLog (208443 => 208444)
--- trunk/Source/WebCore/ChangeLog 2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/ChangeLog 2016-11-09 18:27:37 UTC (rev 208444)
@@ -1,3 +1,33 @@
+2016-11-09 Eric Carlson <[email protected]>
+
+ [MediaStream][Mac] Mark captured video frames as ready for display immediately
+ https://bugs.webkit.org/show_bug.cgi?id=164482
+ <rdar://problem/29139073>
+
+ Reviewed by Jer Noble.
+
+ * platform/cf/CoreMediaSoftLink.cpp: Add new constant.
+ * platform/cf/CoreMediaSoftLink.h:
+
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::requestNotificationWhenReadyForMediaData):
+ New, ask register for a callback when the sample buffer display layer is ready
+ for more media data.
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBuffer): Don't change
+ the sample timestamps, assume the caller has configured the sample correctly.
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::prepareVideoSampleBufferFromTrack): Don't
+ drop frames when the display layer isn't ready.
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::destroyLayer): Call stopRequestingMediaData.
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::sampleBufferUpdated):
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack): Deleted.
+
+ * platform/mediastream/mac/AVVideoCaptureSource.mm:
+ (WebCore::AVVideoCaptureSource::setupCaptureSession): Tell the video output to always discard
+ late video frames, we don't need them.
+ (WebCore::AVVideoCaptureSource::processNewFrame): Add a kCMSampleAttachmentKey_DisplayImmediately
+ attachment.
+
2016-11-09 Joanmarie Diggs <[email protected]>
AX: [ATK] Wrong selected element at a given index in a list box (redux)
Modified: trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.cpp (208443 => 208444)
--- trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.cpp 2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.cpp 2016-11-09 18:27:37 UTC (rev 208444)
@@ -104,6 +104,8 @@
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_DrainAfterDecoding, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_EmptyMedia, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_ResetDecoderBeforeDecoding, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleAttachmentKey_DisplayImmediately, CFStringRef)
+
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMTimebaseNotification_EffectiveRateChanged, CFStringRef)
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMTimebaseNotification_TimeJumped, CFStringRef)
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMAudioFormatDescriptionGetStreamBasicDescription, const AudioStreamBasicDescription *, (CMAudioFormatDescriptionRef desc), (desc))
Modified: trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.h (208443 => 208444)
--- trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.h 2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.h 2016-11-09 18:27:37 UTC (rev 208444)
@@ -166,6 +166,8 @@
#define kCMSampleAttachmentKey_DoNotDisplay get_CoreMedia_kCMSampleAttachmentKey_DoNotDisplay()
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleAttachmentKey_NotSync, CFStringRef)
#define kCMSampleAttachmentKey_NotSync get_CoreMedia_kCMSampleAttachmentKey_NotSync()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleAttachmentKey_DisplayImmediately, CFStringRef)
+#define kCMSampleAttachmentKey_DisplayImmediately get_CoreMedia_kCMSampleAttachmentKey_DisplayImmediately()
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_DisplayEmptyMediaImmediately, CFStringRef)
#define kCMSampleBufferAttachmentKey_DisplayEmptyMediaImmediately get_CoreMedia_kCMSampleBufferAttachmentKey_DisplayEmptyMediaImmediately()
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_DrainAfterDecoding, CFStringRef)
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h (208443 => 208444)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h 2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h 2016-11-09 18:27:37 UTC (rev 208444)
@@ -123,9 +123,12 @@
void setSize(const IntSize&) override { /* No-op */ }
void enqueueAudioSampleBufferFromTrack(MediaStreamTrackPrivate&, MediaSample&);
- void enqueueVideoSampleBufferFromTrack(MediaStreamTrackPrivate&, MediaSample&);
+
+ void prepareVideoSampleBufferFromTrack(MediaStreamTrackPrivate&, MediaSample&);
+ void enqueueVideoSampleBuffer(MediaSample&);
bool shouldEnqueueVideoSampleBuffer() const;
void flushAndRemoveVideoSampleBuffers();
+ void requestNotificationWhenReadyForMediaData();
void paint(GraphicsContext&, const FloatRect&) override;
void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
@@ -194,6 +197,7 @@
HashMap<String, RefPtr<AudioTrackPrivateMediaStream>> m_audioTrackMap;
HashMap<String, RefPtr<VideoTrackPrivateMediaStream>> m_videoTrackMap;
+ Deque<Ref<MediaSample>> m_sampleQueue;
MediaPlayer::NetworkState m_networkState { MediaPlayer::Empty };
MediaPlayer::ReadyState m_readyState { MediaPlayer::HaveNothing };
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm (208443 => 208444)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm 2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm 2016-11-09 18:27:37 UTC (rev 208444)
@@ -128,12 +128,27 @@
// FIXME: https://bugs.webkit.org/show_bug.cgi?id=159836
}
-void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack(MediaStreamTrackPrivate& track, MediaSample& sample)
+void MediaPlayerPrivateMediaStreamAVFObjC::requestNotificationWhenReadyForMediaData()
{
- if (&track != m_mediaStreamPrivate->activeVideoTrack() || !shouldEnqueueVideoSampleBuffer())
- return;
+ [m_sampleBufferDisplayLayer requestMediaDataWhenReadyOnQueue:dispatch_get_main_queue() usingBlock:^ {
+ [m_sampleBufferDisplayLayer stopRequestingMediaData];
- sample.setTimestamps(toMediaTime(CMTimebaseGetTime([m_synchronizer timebase])), MediaTime::invalidTime());
+ while (!m_sampleQueue.isEmpty()) {
+ if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
+ requestNotificationWhenReadyForMediaData();
+ return;
+ }
+
+ auto sample = m_sampleQueue.takeFirst();
+ enqueueVideoSampleBuffer(sample.get());
+ }
+ }];
+}
+
+void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBuffer(MediaSample& sample)
+{
+ ASSERT([m_sampleBufferDisplayLayer isReadyForMoreMediaData]);
+
[m_sampleBufferDisplayLayer enqueueSampleBuffer:sample.platformSample().sample.cmSampleBuffer];
m_isFrameDisplayed = true;
@@ -144,11 +159,22 @@
}
}
+void MediaPlayerPrivateMediaStreamAVFObjC::prepareVideoSampleBufferFromTrack(MediaStreamTrackPrivate& track, MediaSample& sample)
+{
+ if (&track != m_mediaStreamPrivate->activeVideoTrack() || !shouldEnqueueVideoSampleBuffer())
+ return;
+
+ if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
+ m_sampleQueue.append(sample);
+ requestNotificationWhenReadyForMediaData();
+ return;
+ }
+
+ enqueueVideoSampleBuffer(sample);
+}
+
bool MediaPlayerPrivateMediaStreamAVFObjC::shouldEnqueueVideoSampleBuffer() const
{
- if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData])
- return false;
-
if (m_displayMode == LivePreview)
return true;
@@ -189,6 +215,7 @@
if (!m_sampleBufferDisplayLayer)
return;
+ [m_sampleBufferDisplayLayer stopRequestingMediaData];
[m_sampleBufferDisplayLayer flush];
CMTime currentTime = CMTimebaseGetTime([m_synchronizer timebase]);
[m_synchronizer removeRenderer:m_sampleBufferDisplayLayer.get() atTime:currentTime withCompletionHandler:^(BOOL){
@@ -525,7 +552,7 @@
// FIXME: https://bugs.webkit.org/show_bug.cgi?id=159836
break;
case RealtimeMediaSource::Video:
- enqueueVideoSampleBufferFromTrack(track, mediaSample);
+ prepareVideoSampleBufferFromTrack(track, mediaSample);
m_hasReceivedMedia = true;
scheduleDeferredTask([this] {
updateReadyState();
Modified: trunk/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm (208443 => 208444)
--- trunk/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm 2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm 2016-11-09 18:27:37 UTC (rev 208444)
@@ -205,6 +205,7 @@
[NSNumber numberWithInt:kCVPixelFormatType_32BGRA], kCVPixelBufferPixelFormatTypeKey
, nil]);
[videoOutput setVideoSettings:settingsDictionary.get()];
+ [videoOutput setAlwaysDiscardsLateVideoFrames:YES];
setVideoSampleBufferDelegate(videoOutput.get());
if (![session() canAddOutput:videoOutput.get()]) {
@@ -258,11 +259,20 @@
updateFramerate(sampleBuffer.get());
- bool settingsChanged = false;
+ CMSampleBufferRef newSampleBuffer = 0;
+ CMSampleBufferCreateCopy(kCFAllocatorDefault, sampleBuffer.get(), &newSampleBuffer);
+ ASSERT(newSampleBuffer);
- m_buffer = sampleBuffer;
+ CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(newSampleBuffer, true);
+ for (CFIndex i = 0; i < CFArrayGetCount(attachmentsArray); ++i) {
+ CFMutableDictionaryRef attachments = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachmentsArray, i);
+ CFDictionarySetValue(attachments, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
+ }
+
+ m_buffer = newSampleBuffer;
m_lastImage = nullptr;
+ bool settingsChanged = false;
CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
if (dimensions.width != m_width || dimensions.height != m_height) {
m_width = dimensions.width;
@@ -273,7 +283,7 @@
if (settingsChanged)
settingsDidChange();
- mediaDataUpdated(MediaSampleAVFObjC::create(sampleBuffer.get()));
+ mediaDataUpdated(MediaSampleAVFObjC::create(m_buffer.get()));
}
void AVVideoCaptureSource::captureOutputDidOutputSampleBufferFromConnection(AVCaptureOutputType*, CMSampleBufferRef sampleBuffer, AVCaptureConnectionType*)