Title: [208444] trunk/Source/WebCore
Revision
208444
Author
[email protected]
Date
2016-11-09 10:27:37 -0800 (Wed, 09 Nov 2016)

Log Message

[MediaStream][Mac] Mark captured video frames as ready for display immediately
https://bugs.webkit.org/show_bug.cgi?id=164482
<rdar://problem/29139073>

Reviewed by Jer Noble.

* platform/cf/CoreMediaSoftLink.cpp: Add new constant.
* platform/cf/CoreMediaSoftLink.h:

* platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
* platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::requestNotificationWhenReadyForMediaData):
  New, ask register for a callback when the sample buffer display layer is ready
  for more media data.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBuffer): Don't change
  the sample timestamps, assume the caller has configured the sample correctly.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::prepareVideoSampleBufferFromTrack): Don't
  drop frames when the display layer isn't ready.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::destroyLayer): Call stopRequestingMediaData.
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::sampleBufferUpdated):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack): Deleted.

* platform/mediastream/mac/AVVideoCaptureSource.mm:
(WebCore::AVVideoCaptureSource::setupCaptureSession): Tell the video output to always discard
  late video frames, we don't need them.
(WebCore::AVVideoCaptureSource::processNewFrame): Add a kCMSampleAttachmentKey_DisplayImmediately
  attachment.

Modified Paths

Diff

Modified: trunk/Source/WebCore/ChangeLog (208443 => 208444)


--- trunk/Source/WebCore/ChangeLog	2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/ChangeLog	2016-11-09 18:27:37 UTC (rev 208444)
@@ -1,3 +1,33 @@
+2016-11-09  Eric Carlson  <[email protected]>
+
+        [MediaStream][Mac] Mark captured video frames as ready for display immediately
+        https://bugs.webkit.org/show_bug.cgi?id=164482
+        <rdar://problem/29139073>
+
+        Reviewed by Jer Noble.
+
+        * platform/cf/CoreMediaSoftLink.cpp: Add new constant.
+        * platform/cf/CoreMediaSoftLink.h:
+
+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::requestNotificationWhenReadyForMediaData):
+          New, ask register for a callback when the sample buffer display layer is ready
+          for more media data.
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBuffer): Don't change
+          the sample timestamps, assume the caller has configured the sample correctly.
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::prepareVideoSampleBufferFromTrack): Don't
+          drop frames when the display layer isn't ready.
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::destroyLayer): Call stopRequestingMediaData.
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::sampleBufferUpdated):
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack): Deleted.
+
+        * platform/mediastream/mac/AVVideoCaptureSource.mm:
+        (WebCore::AVVideoCaptureSource::setupCaptureSession): Tell the video output to always discard
+          late video frames, we don't need them.
+        (WebCore::AVVideoCaptureSource::processNewFrame): Add a kCMSampleAttachmentKey_DisplayImmediately
+          attachment.
+
 2016-11-09  Joanmarie Diggs  <[email protected]>
 
         AX: [ATK] Wrong selected element at a given index in a list box (redux)

Modified: trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.cpp (208443 => 208444)


--- trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.cpp	2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.cpp	2016-11-09 18:27:37 UTC (rev 208444)
@@ -104,6 +104,8 @@
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_DrainAfterDecoding, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_EmptyMedia, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_ResetDecoderBeforeDecoding, CFStringRef)
+SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMSampleAttachmentKey_DisplayImmediately, CFStringRef)
+
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMTimebaseNotification_EffectiveRateChanged, CFStringRef)
 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMTimebaseNotification_TimeJumped, CFStringRef)
 SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMAudioFormatDescriptionGetStreamBasicDescription, const AudioStreamBasicDescription *, (CMAudioFormatDescriptionRef desc), (desc))

Modified: trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.h (208443 => 208444)


--- trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.h	2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.h	2016-11-09 18:27:37 UTC (rev 208444)
@@ -166,6 +166,8 @@
 #define kCMSampleAttachmentKey_DoNotDisplay get_CoreMedia_kCMSampleAttachmentKey_DoNotDisplay()
 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleAttachmentKey_NotSync, CFStringRef)
 #define kCMSampleAttachmentKey_NotSync get_CoreMedia_kCMSampleAttachmentKey_NotSync()
+SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleAttachmentKey_DisplayImmediately, CFStringRef)
+#define kCMSampleAttachmentKey_DisplayImmediately get_CoreMedia_kCMSampleAttachmentKey_DisplayImmediately()
 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_DisplayEmptyMediaImmediately, CFStringRef)
 #define kCMSampleBufferAttachmentKey_DisplayEmptyMediaImmediately get_CoreMedia_kCMSampleBufferAttachmentKey_DisplayEmptyMediaImmediately()
 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMSampleBufferAttachmentKey_DrainAfterDecoding, CFStringRef)

Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h (208443 => 208444)


--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h	2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h	2016-11-09 18:27:37 UTC (rev 208444)
@@ -123,9 +123,12 @@
     void setSize(const IntSize&) override { /* No-op */ }
 
     void enqueueAudioSampleBufferFromTrack(MediaStreamTrackPrivate&, MediaSample&);
-    void enqueueVideoSampleBufferFromTrack(MediaStreamTrackPrivate&, MediaSample&);
+
+    void prepareVideoSampleBufferFromTrack(MediaStreamTrackPrivate&, MediaSample&);
+    void enqueueVideoSampleBuffer(MediaSample&);
     bool shouldEnqueueVideoSampleBuffer() const;
     void flushAndRemoveVideoSampleBuffers();
+    void requestNotificationWhenReadyForMediaData();
 
     void paint(GraphicsContext&, const FloatRect&) override;
     void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&) override;
@@ -194,6 +197,7 @@
 
     HashMap<String, RefPtr<AudioTrackPrivateMediaStream>> m_audioTrackMap;
     HashMap<String, RefPtr<VideoTrackPrivateMediaStream>> m_videoTrackMap;
+    Deque<Ref<MediaSample>> m_sampleQueue;
 
     MediaPlayer::NetworkState m_networkState { MediaPlayer::Empty };
     MediaPlayer::ReadyState m_readyState { MediaPlayer::HaveNothing };

Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm (208443 => 208444)


--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm	2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm	2016-11-09 18:27:37 UTC (rev 208444)
@@ -128,12 +128,27 @@
     // FIXME: https://bugs.webkit.org/show_bug.cgi?id=159836
 }
 
-void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBufferFromTrack(MediaStreamTrackPrivate& track, MediaSample& sample)
+void MediaPlayerPrivateMediaStreamAVFObjC::requestNotificationWhenReadyForMediaData()
 {
-    if (&track != m_mediaStreamPrivate->activeVideoTrack() || !shouldEnqueueVideoSampleBuffer())
-        return;
+    [m_sampleBufferDisplayLayer requestMediaDataWhenReadyOnQueue:dispatch_get_main_queue() usingBlock:^ {
+        [m_sampleBufferDisplayLayer stopRequestingMediaData];
 
-    sample.setTimestamps(toMediaTime(CMTimebaseGetTime([m_synchronizer timebase])), MediaTime::invalidTime());
+        while (!m_sampleQueue.isEmpty()) {
+            if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
+                requestNotificationWhenReadyForMediaData();
+                return;
+            }
+
+            auto sample = m_sampleQueue.takeFirst();
+            enqueueVideoSampleBuffer(sample.get());
+        }
+    }];
+}
+
+void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSampleBuffer(MediaSample& sample)
+{
+    ASSERT([m_sampleBufferDisplayLayer isReadyForMoreMediaData]);
+
     [m_sampleBufferDisplayLayer enqueueSampleBuffer:sample.platformSample().sample.cmSampleBuffer];
     m_isFrameDisplayed = true;
 
@@ -144,11 +159,22 @@
     }
 }
 
+void MediaPlayerPrivateMediaStreamAVFObjC::prepareVideoSampleBufferFromTrack(MediaStreamTrackPrivate& track, MediaSample& sample)
+{
+    if (&track != m_mediaStreamPrivate->activeVideoTrack() || !shouldEnqueueVideoSampleBuffer())
+        return;
+
+    if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData]) {
+        m_sampleQueue.append(sample);
+        requestNotificationWhenReadyForMediaData();
+        return;
+    }
+
+    enqueueVideoSampleBuffer(sample);
+}
+
 bool MediaPlayerPrivateMediaStreamAVFObjC::shouldEnqueueVideoSampleBuffer() const
 {
-    if (![m_sampleBufferDisplayLayer isReadyForMoreMediaData])
-        return false;
-
     if (m_displayMode == LivePreview)
         return true;
 
@@ -189,6 +215,7 @@
     if (!m_sampleBufferDisplayLayer)
         return;
     
+    [m_sampleBufferDisplayLayer stopRequestingMediaData];
     [m_sampleBufferDisplayLayer flush];
     CMTime currentTime = CMTimebaseGetTime([m_synchronizer timebase]);
     [m_synchronizer removeRenderer:m_sampleBufferDisplayLayer.get() atTime:currentTime withCompletionHandler:^(BOOL){
@@ -525,7 +552,7 @@
         // FIXME: https://bugs.webkit.org/show_bug.cgi?id=159836
         break;
     case RealtimeMediaSource::Video:
-        enqueueVideoSampleBufferFromTrack(track, mediaSample);
+        prepareVideoSampleBufferFromTrack(track, mediaSample);
         m_hasReceivedMedia = true;
         scheduleDeferredTask([this] {
             updateReadyState();

Modified: trunk/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm (208443 => 208444)


--- trunk/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm	2016-11-09 18:26:04 UTC (rev 208443)
+++ trunk/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm	2016-11-09 18:27:37 UTC (rev 208444)
@@ -205,6 +205,7 @@
                                                          [NSNumber numberWithInt:kCVPixelFormatType_32BGRA], kCVPixelBufferPixelFormatTypeKey
                                                          , nil]);
     [videoOutput setVideoSettings:settingsDictionary.get()];
+    [videoOutput setAlwaysDiscardsLateVideoFrames:YES];
     setVideoSampleBufferDelegate(videoOutput.get());
 
     if (![session() canAddOutput:videoOutput.get()]) {
@@ -258,11 +259,20 @@
 
     updateFramerate(sampleBuffer.get());
 
-    bool settingsChanged = false;
+    CMSampleBufferRef newSampleBuffer = 0;
+    CMSampleBufferCreateCopy(kCFAllocatorDefault, sampleBuffer.get(), &newSampleBuffer);
+    ASSERT(newSampleBuffer);
 
-    m_buffer = sampleBuffer;
+    CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(newSampleBuffer, true);
+    for (CFIndex i = 0; i < CFArrayGetCount(attachmentsArray); ++i) {
+        CFMutableDictionaryRef attachments = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachmentsArray, i);
+        CFDictionarySetValue(attachments, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
+    }
+
+    m_buffer = newSampleBuffer;
     m_lastImage = nullptr;
 
+    bool settingsChanged = false;
     CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
     if (dimensions.width != m_width || dimensions.height != m_height) {
         m_width = dimensions.width;
@@ -273,7 +283,7 @@
     if (settingsChanged)
         settingsDidChange();
 
-    mediaDataUpdated(MediaSampleAVFObjC::create(sampleBuffer.get()));
+    mediaDataUpdated(MediaSampleAVFObjC::create(m_buffer.get()));
 }
 
 void AVVideoCaptureSource::captureOutputDidOutputSampleBufferFromConnection(AVCaptureOutputType*, CMSampleBufferRef sampleBuffer, AVCaptureConnectionType*)
_______________________________________________
webkit-changes mailing list
[email protected]
https://lists.webkit.org/mailman/listinfo/webkit-changes

Reply via email to