Diff
Modified: trunk/LayoutTests/imported/w3c/ChangeLog (261552 => 261553)
--- trunk/LayoutTests/imported/w3c/ChangeLog 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/LayoutTests/imported/w3c/ChangeLog 2020-05-12 11:45:25 UTC (rev 261553)
@@ -1,3 +1,14 @@
+2020-05-12 Youenn Fablet <[email protected]>
+
+ Introduce a RealtimeMediaSource video sample observer
+ https://bugs.webkit.org/show_bug.cgi?id=211718
+
+ Reviewed by Eric Carlson.
+
+ * web-platform-tests/mediacapture-streams/MediaStreamTrack-MediaElement-disabled-video-is-black.https-expected.txt:
+ Drive-by fix, since we are now receiving samples but not rendering them when track is not enabled, the loadeddata event is fired appropriately.
+ Update MediaPlayerPrivateMediaStreamAVFObjC::currentDisplayMode to return PaintItBlack in that case.
+
2020-05-11 Antoine Quint <[email protected]>
[Web Animations] Document.getAnimations() should only consider document connection and not timeline association
Modified: trunk/LayoutTests/imported/w3c/web-platform-tests/mediacapture-streams/MediaStreamTrack-MediaElement-disabled-video-is-black.https-expected.txt (261552 => 261553)
--- trunk/LayoutTests/imported/w3c/web-platform-tests/mediacapture-streams/MediaStreamTrack-MediaElement-disabled-video-is-black.https-expected.txt 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/LayoutTests/imported/w3c/web-platform-tests/mediacapture-streams/MediaStreamTrack-MediaElement-disabled-video-is-black.https-expected.txt 2020-05-12 11:45:25 UTC (rev 261553)
@@ -6,7 +6,5 @@
-Harness Error (TIMEOUT), message = null
+PASS Tests that a disabled video track in a MediaStream is rendered as blackness
-TIMEOUT Tests that a disabled video track in a MediaStream is rendered as blackness Test timed out
-
Modified: trunk/Source/WebCore/ChangeLog (261552 => 261553)
--- trunk/Source/WebCore/ChangeLog 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/ChangeLog 2020-05-12 11:45:25 UTC (rev 261553)
@@ -1,3 +1,94 @@
+2020-05-12 Youenn Fablet <[email protected]>
+
+ Introduce a RealtimeMediaSource video sample observer
+ https://bugs.webkit.org/show_bug.cgi?id=211718
+
+ Reviewed by Eric Carlson.
+
+ We introduce an observer dedicated to video samples similarly to AudioSampleObserver for audio.
+ This will allow to move video frame processing out of the main thread progressively.
+ For now, we remove video sample observing from the track private and observers should now register directly to the realtime media source.
+ We update the various users, including MediaRecorder and the media player.
+ In both cases, they will only observe the video track to be played/recorded, which is both more efficient and simpler.
+
+ We introduced RealtimeMediaSource::Observer::hasStartedProducingData callback for MediaStreamTrackPrivate so
+ that the processing to do when the first samples are available can be done on the main thread.
+
+ MediaStreamTrackPrivate no longer filters out samples if it is not enabled.
+ As such, each consumer is now responsible to observe/unobserve the source when its track gets enabled/disabled similarly as for audio,
+ or do nothing if track is not enabled.
+
+ Similarly, RealtimeOutgoingVideoSourceCocoa will now only observe video samples when the track is enabled and not muted.
+
+ Covered by existing tests.
+
+ * Modules/mediarecorder/MediaRecorder.cpp:
+ (WebCore::MediaRecorder::sampleBufferUpdated): Deleted.
+ * Modules/mediarecorder/MediaRecorder.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::~MediaPlayerPrivateMediaStreamAVFObjC):
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSample):
+ We can renove the track check since we only observe the active video track.
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::videoSampleAvailable):
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::checkSelectedVideoTrack):
+ (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::sampleBufferUpdated): Deleted.
+ * platform/mediarecorder/MediaRecorderPrivate.h:
+ (WebCore::MediaRecorderPrivate::setVideoSource):
+ (WebCore::MediaRecorderPrivate::~MediaRecorderPrivate):
+ * platform/mediarecorder/MediaRecorderPrivateAVFImpl.cpp:
+ (WebCore::MediaRecorderPrivateAVFImpl::create):
+ (WebCore::MediaRecorderPrivateAVFImpl::~MediaRecorderPrivateAVFImpl):
+ (WebCore::MediaRecorderPrivateAVFImpl::videoSampleAvailable):
+ (WebCore::MediaRecorderPrivateAVFImpl::stopRecording):
+ (WebCore::MediaRecorderPrivateAVFImpl::sampleBufferUpdated): Deleted.
+ * platform/mediarecorder/MediaRecorderPrivateAVFImpl.h:
+ * platform/mediarecorder/MediaRecorderPrivateMock.cpp:
+ (WebCore::MediaRecorderPrivateMock::MediaRecorderPrivateMock):
+ (WebCore::MediaRecorderPrivateMock::~MediaRecorderPrivateMock):
+ (WebCore::MediaRecorderPrivateMock::stopRecording):
+ (WebCore::MediaRecorderPrivateMock::videoSampleAvailable):
+ (WebCore::MediaRecorderPrivateMock::sampleBufferUpdated): Deleted.
+ * platform/mediarecorder/MediaRecorderPrivateMock.h:
+ * platform/mediastream/MediaStreamTrackPrivate.cpp:
+ (WebCore::MediaStreamTrackPrivate::hasStartedProducingData):
+ (WebCore::MediaStreamTrackPrivate::updateReadyState):
+ (WebCore::MediaStreamTrackPrivate::videoSampleAvailable): Deleted.
+ (WebCore::MediaStreamTrackPrivate::hasStartedProducingAudioData): Deleted.
+ * platform/mediastream/MediaStreamTrackPrivate.h:
+ * platform/mediastream/RealtimeMediaSource.cpp:
+ (WebCore::RealtimeMediaSource::addVideoSampleObserver):
+ (WebCore::RealtimeMediaSource::removeVideoSampleObserver):
+ (WebCore::RealtimeMediaSource::updateHasStartedProducingData):
+ (WebCore::RealtimeMediaSource::videoSampleAvailable):
+ (WebCore::RealtimeMediaSource::audioSamplesAvailable):
+ * platform/mediastream/RealtimeMediaSource.h:
+ * platform/mediastream/RealtimeOutgoingVideoSource.cpp:
+ (WebCore::RealtimeOutgoingVideoSource::unobserveSource):
+ (WebCore::RealtimeOutgoingVideoSource::updateBlackFramesSending):
+ * platform/mediastream/RealtimeOutgoingVideoSource.h:
+ * platform/mediastream/RealtimeVideoSource.cpp:
+ (WebCore::RealtimeVideoSource::~RealtimeVideoSource):
+ (WebCore::RealtimeVideoSource::startProducingData):
+ (WebCore::RealtimeVideoSource::stopProducingData):
+ (WebCore::RealtimeVideoSource::videoSampleAvailable):
+ * platform/mediastream/RealtimeVideoSource.h:
+ * platform/mediastream/mac/AVVideoCaptureSource.mm:
+ (WebCore::AVVideoCaptureSource::captureOutputDidOutputSampleBufferFromConnection):
+ * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h:
+ * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm:
+ (WebCore::RealtimeIncomingVideoSourceCocoa::processNewSample):
+ * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp:
+ (WebCore::RealtimeOutgoingVideoSourceCocoa::videoSampleAvailable):
+ (WebCore::RealtimeOutgoingVideoSourceCocoa::sampleBufferUpdated): Deleted.
+ * platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h:
+ * platform/mock/MockRealtimeVideoSource.cpp:
+ * testing/Internals.cpp:
+ (WebCore::Internals::~Internals):
+ (WebCore::Internals::stopObservingRealtimeMediaSource):
+ (WebCore::Internals::observeMediaStreamTrack):
+ * testing/Internals.h:
+
2020-05-12 Philippe Normand <[email protected]>
[GStreamer] Audio messages in web.whatsapp.com only play once.
Modified: trunk/Source/WebCore/Modules/mediarecorder/MediaRecorder.cpp (261552 => 261553)
--- trunk/Source/WebCore/Modules/mediarecorder/MediaRecorder.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/Modules/mediarecorder/MediaRecorder.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -222,11 +222,6 @@
stopRecording();
}
-void MediaRecorder::sampleBufferUpdated(MediaStreamTrackPrivate& track, MediaSample& mediaSample)
-{
- m_private->sampleBufferUpdated(track, mediaSample);
-}
-
bool MediaRecorder::virtualHasPendingActivity() const
{
return m_state != RecordingState::Inactive;
Modified: trunk/Source/WebCore/Modules/mediarecorder/MediaRecorder.h (261552 => 261553)
--- trunk/Source/WebCore/Modules/mediarecorder/MediaRecorder.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/Modules/mediarecorder/MediaRecorder.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -108,7 +108,6 @@
void trackMutedChanged(MediaStreamTrackPrivate&) final { };
void trackSettingsChanged(MediaStreamTrackPrivate&) final { };
void trackEnabledChanged(MediaStreamTrackPrivate&) final { };
- void sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample&) final;
static CreatorFunction m_customCreator;
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h (261552 => 261553)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -51,7 +51,12 @@
class VideoLayerManagerObjC;
class VideoTrackPrivateMediaStream;
-class MediaPlayerPrivateMediaStreamAVFObjC final : public MediaPlayerPrivateInterface, private MediaStreamPrivate::Observer, public MediaStreamTrackPrivate::Observer, public SampleBufferDisplayLayer::Client
+class MediaPlayerPrivateMediaStreamAVFObjC final
+ : public MediaPlayerPrivateInterface
+ , private MediaStreamPrivate::Observer
+ , public MediaStreamTrackPrivate::Observer
+ , public RealtimeMediaSource::VideoSampleObserver
+ , public SampleBufferDisplayLayer::Client
, private LoggerHelper
{
public:
@@ -134,7 +139,7 @@
MediaTime calculateTimelineOffset(const MediaSample&, double);
- void enqueueVideoSample(MediaStreamTrackPrivate&, MediaSample&);
+ void enqueueVideoSample(MediaSample&);
void enqueueCorrectedVideoSample(MediaSample&);
void requestNotificationWhenReadyForVideoData();
@@ -197,9 +202,11 @@
void trackMutedChanged(MediaStreamTrackPrivate&) override { };
void trackSettingsChanged(MediaStreamTrackPrivate&) override { };
void trackEnabledChanged(MediaStreamTrackPrivate&) override { };
- void sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample&) override;
void readyStateChanged(MediaStreamTrackPrivate&) override;
+ // RealtimeMediaSouce::VideoSampleObserver
+ void videoSampleAvailable(MediaSample&) final;
+
RetainPtr<PlatformLayer> createVideoFullscreenLayer() override;
void setVideoFullscreenLayer(PlatformLayer*, WTF::Function<void()>&& completionHandler) override;
void setVideoFullscreenFrame(FloatRect) override;
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm (261552 => 261553)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm 2020-05-12 11:45:25 UTC (rev 261553)
@@ -155,16 +155,18 @@
for (const auto& track : m_audioTrackMap.values())
track->pause();
- if (m_mediaStreamPrivate) {
+ if (m_mediaStreamPrivate)
m_mediaStreamPrivate->removeObserver(*this);
- for (auto& track : m_audioTrackMap.values())
- track->streamTrack().removeObserver(*this);
+ for (auto& track : m_audioTrackMap.values())
+ track->streamTrack().removeObserver(*this);
- for (auto& track : m_videoTrackMap.values())
- track->streamTrack().removeObserver(*this);
- }
+ for (auto& track : m_videoTrackMap.values())
+ track->streamTrack().removeObserver(*this);
+ if (m_activeVideoTrack)
+ m_activeVideoTrack->source().removeVideoSampleObserver(*this);
+
[m_boundsChangeListener invalidate];
destroyLayers();
@@ -279,11 +281,8 @@
}
}
-void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSample(MediaStreamTrackPrivate& track, MediaSample& sample)
+void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSample(MediaSample& sample)
{
- if (&track != activeVideoTrack())
- return;
-
if (!m_imagePainter.mediaSample || m_displayMode != PausedImage) {
m_imagePainter.mediaSample = &sample;
m_imagePainter.cgImage = nullptr;
@@ -294,13 +293,14 @@
if (m_displayMode != LivePreview && !m_waitingForFirstImage)
return;
- auto videoTrack = m_videoTrackMap.get(track.id());
+ // FIXME: We should not query the map each time we get a sample.
+ auto videoTrack = m_videoTrackMap.get(m_activeVideoTrack->id());
MediaTime timelineOffset = videoTrack->timelineOffset();
if (timelineOffset == MediaTime::invalidTime()) {
timelineOffset = calculateTimelineOffset(sample, rendererLatency);
videoTrack->setTimelineOffset(timelineOffset);
- INFO_LOG(LOGIDENTIFIER, "timeline offset for track ", track.id(), " set to ", timelineOffset);
+ INFO_LOG(LOGIDENTIFIER, "timeline offset for track ", m_activeVideoTrack->id(), " set to ", timelineOffset);
}
DEBUG_LOG(LOGIDENTIFIER, "original sample = ", sample);
@@ -471,7 +471,7 @@
MediaPlayerPrivateMediaStreamAVFObjC::DisplayMode MediaPlayerPrivateMediaStreamAVFObjC::currentDisplayMode() const
{
- if (m_intrinsicSize.isEmpty() || !metaDataAvailable() || !m_sampleBufferDisplayLayer)
+ if (m_intrinsicSize.isEmpty() || !metaDataAvailable())
return None;
if (auto* track = activeVideoTrack()) {
@@ -735,26 +735,12 @@
updateTracks();
}
-void MediaPlayerPrivateMediaStreamAVFObjC::sampleBufferUpdated(MediaStreamTrackPrivate& track, MediaSample& mediaSample)
+void MediaPlayerPrivateMediaStreamAVFObjC::videoSampleAvailable(MediaSample& mediaSample)
{
- ASSERT(track.id() == mediaSample.trackID());
- ASSERT(mediaSample.platformSample().type == PlatformSample::CMSampleBufferType);
- ASSERT(m_mediaStreamPrivate);
-
if (streamTime().toDouble() < 0)
return;
- switch (track.type()) {
- case RealtimeMediaSource::Type::None:
- // Do nothing.
- break;
- case RealtimeMediaSource::Type::Audio:
- break;
- case RealtimeMediaSource::Type::Video:
- if (&track == m_activeVideoTrack.get())
- enqueueVideoSample(track, mediaSample);
- break;
- }
+ enqueueVideoSample(mediaSample);
}
void MediaPlayerPrivateMediaStreamAVFObjC::readyStateChanged(MediaStreamTrackPrivate&)
@@ -851,6 +837,7 @@
scheduleDeferredTask([this] {
auto oldVideoTrack = m_activeVideoTrack;
bool hideVideoLayer = true;
+
m_activeVideoTrack = nullptr;
if (auto* activeVideoTrack = this->activeVideoTrack()) {
for (const auto& track : m_videoTrackMap.values()) {
@@ -877,6 +864,13 @@
m_pendingSelectedTrackCheck = false;
updateDisplayMode();
+
+ if (oldVideoTrack != m_activeVideoTrack) {
+ if (oldVideoTrack)
+ oldVideoTrack->source().removeVideoSampleObserver(*this);
+ if (m_activeVideoTrack)
+ m_activeVideoTrack->source().addVideoSampleObserver(*this);
+ }
});
}
Modified: trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivate.h (261552 => 261553)
--- trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivate.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivate.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -44,8 +44,9 @@
class PlatformAudioData;
class SharedBuffer;
-class MediaRecorderPrivate :
- public RealtimeMediaSource::AudioSampleObserver {
+class MediaRecorderPrivate
+ : public RealtimeMediaSource::AudioSampleObserver
+ , public RealtimeMediaSource::VideoSampleObserver {
public:
~MediaRecorderPrivate();
@@ -55,8 +56,6 @@
};
WEBCORE_EXPORT static AudioVideoSelectedTracks selectTracks(MediaStreamPrivate&);
- virtual void sampleBufferUpdated(const MediaStreamTrackPrivate&, MediaSample&) = 0;
-
using FetchDataCallback = CompletionHandler<void(RefPtr<SharedBuffer>&&, const String& mimeType)>;
virtual void fetchData(FetchDataCallback&&) = 0;
virtual void stopRecording() = 0;
@@ -66,6 +65,7 @@
protected:
void setAudioSource(RefPtr<RealtimeMediaSource>&&);
+ void setVideoSource(RefPtr<RealtimeMediaSource>&&);
protected:
ErrorCallback m_errorCallback;
@@ -72,6 +72,7 @@
private:
RefPtr<RealtimeMediaSource> m_audioSource;
+ RefPtr<RealtimeMediaSource> m_videoSource;
};
inline void MediaRecorderPrivate::setAudioSource(RefPtr<RealtimeMediaSource>&& audioSource)
@@ -85,10 +86,26 @@
m_audioSource->addAudioSampleObserver(*this);
}
+inline void MediaRecorderPrivate::setVideoSource(RefPtr<RealtimeMediaSource>&& videoSource)
+{
+ if (m_videoSource)
+ m_videoSource->removeVideoSampleObserver(*this);
+
+ m_videoSource = WTFMove(videoSource);
+
+ if (m_videoSource)
+ m_videoSource->addVideoSampleObserver(*this);
+}
+
inline MediaRecorderPrivate::~MediaRecorderPrivate()
{
+ // Subclasses should stop observing sonner than here. Otherwise they might be called from a background thread while half destroyed
+ ASSERT(!m_audioSource);
+ ASSERT(!m_videoSource);
if (m_audioSource)
m_audioSource->removeAudioSampleObserver(*this);
+ if (m_videoSource)
+ m_videoSource->removeVideoSampleObserver(*this);
}
} // namespace WebCore
Modified: trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.cpp (261552 => 261553)
--- trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -59,6 +59,8 @@
auto recorder = makeUnique<MediaRecorderPrivateAVFImpl>(writer.releaseNonNull(), WTFMove(audioTrackId), WTFMove(videoTrackId));
if (selectedTracks.audioTrack)
recorder->setAudioSource(&selectedTracks.audioTrack->source());
+ if (selectedTracks.videoTrack)
+ recorder->setVideoSource(&selectedTracks.videoTrack->source());
return recorder;
}
@@ -72,12 +74,11 @@
MediaRecorderPrivateAVFImpl::~MediaRecorderPrivateAVFImpl()
{
setAudioSource(nullptr);
+ setVideoSource(nullptr);
}
-void MediaRecorderPrivateAVFImpl::sampleBufferUpdated(const MediaStreamTrackPrivate& track, MediaSample& sampleBuffer)
+void MediaRecorderPrivateAVFImpl::videoSampleAvailable(MediaSample& sampleBuffer)
{
- if (track.id() != m_recordedVideoTrackID)
- return;
m_writer->appendVideoSampleBuffer(sampleBuffer.platformSample().sample.cmSampleBuffer);
}
@@ -91,6 +92,7 @@
void MediaRecorderPrivateAVFImpl::stopRecording()
{
setAudioSource(nullptr);
+ setVideoSource(nullptr);
m_writer->stopRecording();
}
Modified: trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.h (261552 => 261553)
--- trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateAVFImpl.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -46,7 +46,7 @@
friend std::unique_ptr<MediaRecorderPrivateAVFImpl> std::make_unique<MediaRecorderPrivateAVFImpl>(Ref<MediaRecorderPrivateWriter>&&, String&&, String&&);
// MediaRecorderPrivate
- void sampleBufferUpdated(const MediaStreamTrackPrivate&, MediaSample&) final;
+ void videoSampleAvailable(MediaSample&) final;
void fetchData(FetchDataCallback&&) final;
void audioSamplesAvailable(const WTF::MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t) final;
Modified: trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.cpp (261552 => 261553)
--- trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -40,21 +40,25 @@
m_audioTrackID = selectedTracks.audioTrack->id();
setAudioSource(&selectedTracks.audioTrack->source());
}
- if (selectedTracks.videoTrack)
+ if (selectedTracks.videoTrack) {
m_videoTrackID = selectedTracks.videoTrack->id();
+ setVideoSource(&selectedTracks.videoTrack->source());
+ }
}
MediaRecorderPrivateMock::~MediaRecorderPrivateMock()
{
setAudioSource(nullptr);
+ setVideoSource(nullptr);
}
void MediaRecorderPrivateMock::stopRecording()
{
setAudioSource(nullptr);
+ setVideoSource(nullptr);
}
-void MediaRecorderPrivateMock::sampleBufferUpdated(const MediaStreamTrackPrivate&, MediaSample&)
+void MediaRecorderPrivateMock::videoSampleAvailable(MediaSample&)
{
auto locker = holdLock(m_bufferLock);
m_buffer.append("Video Track ID: ");
Modified: trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.h (261552 => 261553)
--- trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediarecorder/MediaRecorderPrivateMock.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -42,7 +42,7 @@
private:
// MediaRecorderPrivate
- void sampleBufferUpdated(const MediaStreamTrackPrivate&, MediaSample&) final;
+ void videoSampleAvailable(MediaSample&) final;
void fetchData(FetchDataCallback&&) final;
void audioSamplesAvailable(const WTF::MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t) final;
void stopRecording() final;
Modified: trunk/Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.cpp (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -236,28 +236,12 @@
return !m_isEnded;
}
-void MediaStreamTrackPrivate::videoSampleAvailable(MediaSample& mediaSample)
+void MediaStreamTrackPrivate::hasStartedProducingData()
{
ASSERT(isMainThread());
- if (!m_haveProducedData) {
- m_haveProducedData = true;
- updateReadyState();
- }
-
- if (!enabled())
+ if (m_hasStartedProducingData)
return;
-
- mediaSample.setTrackID(id());
- forEachObserver([&](auto& observer) {
- observer.sampleBufferUpdated(*this, mediaSample);
- });
-}
-
-void MediaStreamTrackPrivate::hasStartedProducingAudioData()
-{
- if (m_haveProducedData)
- return;
- m_haveProducedData = true;
+ m_hasStartedProducingData = true;
updateReadyState();
}
@@ -267,7 +251,7 @@
if (m_isEnded)
state = ReadyState::Ended;
- else if (m_haveProducedData)
+ else if (m_hasStartedProducingData)
state = ReadyState::Live;
if (state == m_readyState)
Modified: trunk/Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.h (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -59,7 +59,6 @@
virtual void trackMutedChanged(MediaStreamTrackPrivate&) = 0;
virtual void trackSettingsChanged(MediaStreamTrackPrivate&) = 0;
virtual void trackEnabledChanged(MediaStreamTrackPrivate&) = 0;
- virtual void sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample&) { };
virtual void readyStateChanged(MediaStreamTrackPrivate&) { };
};
@@ -133,9 +132,8 @@
void sourceMutedChanged() final;
void sourceSettingsChanged() final;
bool preventSourceFromStopping() final;
- void videoSampleAvailable(MediaSample&) final;
void audioUnitWillStart() final;
- void hasStartedProducingAudioData() final;
+ void hasStartedProducingData() final;
void updateReadyState();
@@ -153,8 +151,7 @@
ReadyState m_readyState { ReadyState::None };
bool m_isEnabled { true };
bool m_isEnded { false };
- bool m_haveProducedData { false };
- bool m_hasSentStartProducedData { false };
+ bool m_hasStartedProducingData { false };
HintValue m_contentHint { HintValue::Empty };
RefPtr<WebAudioSourceProvider> m_audioSourceProvider;
Ref<const Logger> m_logger;
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -74,6 +74,20 @@
m_audioSampleObservers.remove(&observer);
}
+void RealtimeMediaSource::addVideoSampleObserver(VideoSampleObserver& observer)
+{
+ ASSERT(isMainThread());
+ auto locker = holdLock(m_videoSampleObserversLock);
+ m_videoSampleObservers.add(&observer);
+}
+
+void RealtimeMediaSource::removeVideoSampleObserver(VideoSampleObserver& observer)
+{
+ ASSERT(isMainThread());
+ auto locker = holdLock(m_videoSampleObserversLock);
+ m_videoSampleObservers.remove(&observer);
+}
+
void RealtimeMediaSource::addObserver(Observer& observer)
{
ASSERT(isMainThread());
@@ -154,8 +168,27 @@
});
}
+void RealtimeMediaSource::updateHasStartedProducingData()
+{
+ if (m_hasStartedProducingData)
+ return;
+
+ callOnMainThread([this, weakThis = makeWeakPtr(this)] {
+ if (!weakThis)
+ return;
+ if (m_hasStartedProducingData)
+ return;
+ m_hasStartedProducingData = true;
+ forEachObserver([&](auto& observer) {
+ observer.hasStartedProducingData();
+ });
+ });
+}
+
void RealtimeMediaSource::videoSampleAvailable(MediaSample& mediaSample)
{
+ // FIXME: Migrate RealtimeMediaSource clients to non main thread processing.
+ ASSERT(isMainThread());
#if !RELEASE_LOG_DISABLED
++m_frameCount;
@@ -170,25 +203,16 @@
}
#endif
- forEachObserver([&](auto& observer) {
- observer.videoSampleAvailable(mediaSample);
- });
+ updateHasStartedProducingData();
+
+ auto locker = holdLock(m_videoSampleObserversLock);
+ for (auto* observer : m_videoSampleObservers)
+ observer->videoSampleAvailable(mediaSample);
}
void RealtimeMediaSource::audioSamplesAvailable(const MediaTime& time, const PlatformAudioData& audioData, const AudioStreamDescription& description, size_t numberOfFrames)
{
- if (!m_hasSentStartProducedAudioData) {
- callOnMainThread([this, weakThis = makeWeakPtr(this)] {
- if (!weakThis)
- return;
- if (m_hasSentStartProducedAudioData)
- return;
- m_hasSentStartProducedAudioData = true;
- forEachObserver([&](auto& observer) {
- observer.hasStartedProducingAudioData();
- });
- });
- }
+ updateHasStartedProducingData();
auto locker = holdLock(m_audioSampleObserversLock);
for (auto* observer : m_audioSampleObservers)
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.h (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -89,10 +89,7 @@
// Observer state queries.
virtual bool preventSourceFromStopping() { return false; }
- // Called on the main thread.
- virtual void videoSampleAvailable(MediaSample&) { }
-
- virtual void hasStartedProducingAudioData() { }
+ virtual void hasStartedProducingData() { }
};
class AudioSampleObserver {
public:
@@ -101,7 +98,13 @@
// May be called on a background thread.
virtual void audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t /*numberOfFrames*/) = 0;
};
+ class VideoSampleObserver {
+ public:
+ virtual ~VideoSampleObserver() = default;
+ virtual void videoSampleAvailable(MediaSample&) { }
+ };
+
virtual ~RealtimeMediaSource() = default;
virtual Ref<RealtimeMediaSource> clone() { return *this; }
@@ -139,6 +142,9 @@
WEBCORE_EXPORT void addAudioSampleObserver(AudioSampleObserver&);
WEBCORE_EXPORT void removeAudioSampleObserver(AudioSampleObserver&);
+ WEBCORE_EXPORT void addVideoSampleObserver(VideoSampleObserver&);
+ WEBCORE_EXPORT void removeVideoSampleObserver(VideoSampleObserver&);
+
const IntSize size() const;
void setSize(const IntSize&);
@@ -250,6 +256,8 @@
virtual void hasEnded() { }
+ void updateHasStartedProducingData();
+
#if !RELEASE_LOG_DISABLED
RefPtr<const Logger> m_logger;
const void* m_logIdentifier;
@@ -267,6 +275,9 @@
mutable RecursiveLock m_audioSampleObserversLock;
HashSet<AudioSampleObserver*> m_audioSampleObservers;
+ mutable RecursiveLock m_videoSampleObserversLock;
+ HashSet<VideoSampleObserver*> m_videoSampleObservers;
+
IntSize m_size;
IntSize m_intrinsicSize;
double m_frameRate { 30 };
@@ -284,7 +295,7 @@
bool m_interrupted { false };
bool m_captureDidFailed { false };
bool m_isEnded { false };
- bool m_hasSentStartProducedAudioData { false };
+ bool m_hasStartedProducingData { false };
};
struct CaptureSourceOrError {
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingVideoSource.cpp (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingVideoSource.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingVideoSource.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -74,6 +74,7 @@
void RealtimeOutgoingVideoSource::unobserveSource()
{
m_videoSource->removeObserver(*this);
+ m_videoSource->source().removeVideoSampleObserver(*this);
}
void RealtimeOutgoingVideoSource::setSource(Ref<MediaStreamTrackPrivate>&& newSource)
@@ -114,11 +115,13 @@
void RealtimeOutgoingVideoSource::updateBlackFramesSending()
{
if (!m_muted && m_enabled) {
+ m_videoSource->source().addVideoSampleObserver(*this);
if (m_blackFrameTimer.isActive())
m_blackFrameTimer.stop();
return;
}
+ m_videoSource->source().removeVideoSampleObserver(*this);
sendBlackFramesIfNeeded();
}
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingVideoSource.h (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingVideoSource.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingVideoSource.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -51,6 +51,7 @@
: public ThreadSafeRefCounted<RealtimeOutgoingVideoSource, WTF::DestructionThread::Main>
, public webrtc::VideoTrackSourceInterface
, private MediaStreamTrackPrivate::Observer
+ , private RealtimeMediaSource::VideoSampleObserver
#if !RELEASE_LOG_DISABLED
, private LoggerHelper
#endif
@@ -128,9 +129,11 @@
void trackMutedChanged(MediaStreamTrackPrivate&) final { sourceMutedChanged(); }
void trackEnabledChanged(MediaStreamTrackPrivate&) final { sourceEnabledChanged(); }
void trackSettingsChanged(MediaStreamTrackPrivate&) final { initializeFromSource(); }
- void sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample&) override { }
void trackEnded(MediaStreamTrackPrivate&) final { }
+ // RealtimeMediaSource::VideoSampleObserver API
+ void videoSampleAvailable(MediaSample&) override { }
+
Ref<MediaStreamTrackPrivate> m_videoSource;
Timer m_blackFrameTimer;
rtc::scoped_refptr<webrtc::VideoFrameBuffer> m_blackFrame;
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -46,6 +46,7 @@
RealtimeVideoSource::~RealtimeVideoSource()
{
+ m_source->removeVideoSampleObserver(*this);
m_source->removeObserver(*this);
}
@@ -52,10 +53,12 @@
void RealtimeVideoSource::startProducingData()
{
m_source->start();
+ m_source->addVideoSampleObserver(*this);
}
void RealtimeVideoSource::stopProducingData()
{
+ m_source->removeVideoSampleObserver(*this);
m_source->stop();
}
@@ -166,9 +169,6 @@
void RealtimeVideoSource::videoSampleAvailable(MediaSample& sample)
{
- if (!isProducingData())
- return;
-
if (m_frameDecimation > 1 && ++m_frameDecimationCounter % m_frameDecimation)
return;
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.h (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -33,7 +33,10 @@
class ImageTransferSessionVT;
-class RealtimeVideoSource final : public RealtimeMediaSource, public RealtimeMediaSource::Observer {
+class RealtimeVideoSource final
+ : public RealtimeMediaSource
+ , public RealtimeMediaSource::Observer
+ , public RealtimeMediaSource::VideoSampleObserver {
public:
static Ref<RealtimeVideoSource> create(Ref<RealtimeVideoCaptureSource>&& source) { return adoptRef(*new RealtimeVideoSource(WTFMove(source))); }
@@ -58,11 +61,13 @@
bool interrupted() const final { return m_source->interrupted(); }
bool isSameAs(RealtimeMediaSource& source) const final { return this == &source || m_source.ptr() == &source; }
- // Observer
+ // RealtimeMediaSource::Observer
void sourceMutedChanged() final;
void sourceSettingsChanged() final;
void sourceStopped() final;
bool preventSourceFromStopping() final;
+
+ // RealtimeMediaSource::VideoSampleObserver
void videoSampleAvailable(MediaSample&) final;
#if PLATFORM(COCOA)
Modified: trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -107,7 +107,8 @@
class WebKitMediaStreamTrackObserver
: public MediaStreamTrackPrivate::Observer
- , public RealtimeMediaSource::AudioSampleObserver {
+ , public RealtimeMediaSource::AudioSampleObserver
+ , public RealtimeMediaSource::VideoSampleObserver {
WTF_MAKE_FAST_ALLOCATED;
public:
virtual ~WebKitMediaStreamTrackObserver() { };
@@ -130,7 +131,7 @@
void trackSettingsChanged(MediaStreamTrackPrivate&) final { };
void readyStateChanged(MediaStreamTrackPrivate&) final { };
- void sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample& sample) final
+ void videoSampleAvailable(MediaSample& sample) final
{
if (!m_enabled)
return;
@@ -387,6 +388,7 @@
if (self->stream) {
for (auto& track : self->stream->tracks()) {
track->source().removeAudioSampleObserver(*self->mediaStreamTrackObserver.get());
+ track->source().removeVideoSampleObserver(*self->mediaStreamTrackObserver.get());
track->removeObserver(*self->mediaStreamTrackObserver.get());
}
self->stream->removeObserver(*self->mediaStreamObserver);
@@ -409,10 +411,12 @@
if (self->stream) {
for (auto& track : self->stream->tracks()) {
track->source().removeAudioSampleObserver(*self->mediaStreamTrackObserver.get());
+ track->source().removeVideoSampleObserver(*self->mediaStreamTrackObserver.get());
track->removeObserver(*self->mediaStreamTrackObserver.get());
}
} else if (self->track) {
self->track->source().removeAudioSampleObserver(*self->mediaStreamTrackObserver.get());
+ self->track->source().removeVideoSampleObserver(*self->mediaStreamTrackObserver.get());
self->track->removeObserver(*self->mediaStreamTrackObserver.get());
}
GST_OBJECT_UNLOCK(self);
@@ -560,7 +564,17 @@
if (observe_track) {
track->addObserver(*self->mediaStreamTrackObserver.get());
- track->source().addAudioSampleObserver(*self->mediaStreamTrackObserver.get());
+ auto& source = track->source();
+ switch (source.type()) {
+ case RealtimeMediaSource::Type::Audio:
+ source.addAudioSampleObserver(*self->mediaStreamTrackObserver.get());
+ break;
+ case RealtimeMediaSource::Type::Video:
+ source.addVideoSampleObserver(*self->mediaStreamTrackObserver.get());
+ break;
+ case RealtimeMediaSource::Type::None:
+ ASSERT_NOT_REACHED();
+ }
}
gst_element_sync_state_with_parent(element);
return TRUE;
Modified: trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.cpp (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -50,11 +50,8 @@
{
}
-void RealtimeOutgoingVideoSourceLibWebRTC::sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample& sample)
+void RealtimeOutgoingVideoSourceLibWebRTC::videoSampleAvailable(MediaSample& sample)
{
- if (isSilenced())
- return;
-
switch (sample.videoRotation()) {
case MediaSample::VideoRotation::None:
m_currentRotation = webrtc::kVideoRotation_0;
Modified: trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.h (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingVideoSourceLibWebRTC.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -41,8 +41,8 @@
rtc::scoped_refptr<webrtc::VideoFrameBuffer> createBlackFrame(size_t, size_t) final;
- // MediaStreamTrackPrivate::Observer API
- void sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample&) final;
+ // RealtimeMediaSource::VideoSampleObserver API
+ void videoSampleAvailable(MediaSample&) final;
};
} // namespace WebCore
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -50,7 +50,6 @@
// rtc::VideoSinkInterface
void OnFrame(const webrtc::VideoFrame&) final;
- RetainPtr<CMSampleBufferRef> m_buffer;
RetainPtr<CVPixelBufferRef> m_blackFrame;
int m_blackFrameWidth { 0 };
int m_blackFrameHeight { 0 };
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm 2020-05-12 11:45:25 UTC (rev 261553)
@@ -200,7 +200,6 @@
void RealtimeIncomingVideoSourceCocoa::processNewSample(CMSampleBufferRef sample, unsigned width, unsigned height, MediaSample::VideoRotation rotation)
{
- m_buffer = sample;
auto size = this->size();
if (WTF::safeCast<int>(width) != size.width() || WTF::safeCast<int>(height) != size.height())
setIntrinsicSize(IntSize(width, height));
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -62,11 +62,8 @@
{
}
-void RealtimeOutgoingVideoSourceCocoa::sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample& sample)
+void RealtimeOutgoingVideoSourceCocoa::videoSampleAvailable(MediaSample& sample)
{
- if (isSilenced())
- return;
-
#if !RELEASE_LOG_DISABLED
if (!(++m_numberOfFrames % 60))
ALWAYS_LOG(LOGIDENTIFIER, "frame ", m_numberOfFrames);
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h (261552 => 261553)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSourceCocoa.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -44,8 +44,8 @@
rtc::scoped_refptr<webrtc::VideoFrameBuffer> createBlackFrame(size_t width, size_t height) final;
- // MediaStreamTrackPrivate::Observer API
- void sampleBufferUpdated(MediaStreamTrackPrivate&, MediaSample&) final;
+ // RealtimeMediaSource::VideoSampleObserver API
+ void videoSampleAvailable(MediaSample&) final;
RetainPtr<CVPixelBufferRef> convertToYUV(CVPixelBufferRef);
RetainPtr<CVPixelBufferRef> rotatePixelBuffer(CVPixelBufferRef, webrtc::VideoRotation);
Modified: trunk/Source/WebCore/testing/Internals.cpp (261552 => 261553)
--- trunk/Source/WebCore/testing/Internals.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/testing/Internals.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -476,10 +476,7 @@
Internals::~Internals()
{
#if ENABLE(MEDIA_STREAM)
- if (m_trackSource) {
- m_trackSource->removeObserver(*this);
- m_trackSource->removeAudioSampleObserver(*this);
- }
+ stopObservingRealtimeMediaSource();
#endif
}
@@ -5039,19 +5036,44 @@
source.monitorOrientation(m_orientationNotifier);
}
-void Internals::observeMediaStreamTrack(MediaStreamTrack& track)
+void Internals::stopObservingRealtimeMediaSource()
{
- if (m_trackSource) {
- m_trackSource->removeObserver(*this);
+ if (!m_trackSource)
+ return;
+
+ switch (m_trackSource->type()) {
+ case RealtimeMediaSource::Type::Audio:
m_trackSource->removeAudioSampleObserver(*this);
-
- m_trackAudioSampleCount = 0;
- m_trackVideoSampleCount = 0;
+ break;
+ case RealtimeMediaSource::Type::Video:
+ m_trackSource->removeVideoSampleObserver(*this);
+ break;
+ case RealtimeMediaSource::Type::None:
+ ASSERT_NOT_REACHED();
}
+ m_trackSource->removeObserver(*this);
+ m_trackSource = nullptr;
+ m_trackAudioSampleCount = 0;
+ m_trackVideoSampleCount = 0;
+}
+
+void Internals::observeMediaStreamTrack(MediaStreamTrack& track)
+{
+ stopObservingRealtimeMediaSource();
+
m_trackSource = &track.source();
m_trackSource->addObserver(*this);
- m_trackSource->addAudioSampleObserver(*this);
+ switch (m_trackSource->type()) {
+ case RealtimeMediaSource::Type::Audio:
+ m_trackSource->addAudioSampleObserver(*this);
+ break;
+ case RealtimeMediaSource::Type::Video:
+ m_trackSource->addVideoSampleObserver(*this);
+ break;
+ case RealtimeMediaSource::Type::None:
+ ASSERT_NOT_REACHED();
+ }
}
void Internals::grabNextMediaStreamTrackFrame(TrackFramePromise&& promise)
Modified: trunk/Source/WebCore/testing/Internals.h (261552 => 261553)
--- trunk/Source/WebCore/testing/Internals.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebCore/testing/Internals.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -130,6 +130,7 @@
#if ENABLE(MEDIA_STREAM)
, private RealtimeMediaSource::Observer
, private RealtimeMediaSource::AudioSampleObserver
+ , private RealtimeMediaSource::VideoSampleObserver
#endif
{
public:
@@ -763,6 +764,8 @@
#endif
#if ENABLE(MEDIA_STREAM)
+ void stopObservingRealtimeMediaSource();
+
void setMockAudioTrackChannelNumber(MediaStreamTrack&, unsigned short);
void setCameraMediaStreamTrackOrientation(MediaStreamTrack&, int orientation);
unsigned long trackAudioSampleCount() const { return m_trackAudioSampleCount; }
Modified: trunk/Source/WebKit/ChangeLog (261552 => 261553)
--- trunk/Source/WebKit/ChangeLog 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebKit/ChangeLog 2020-05-12 11:45:25 UTC (rev 261553)
@@ -1,3 +1,21 @@
+2020-05-12 Youenn Fablet <[email protected]>
+
+ Introduce a RealtimeMediaSource video sample observer
+ https://bugs.webkit.org/show_bug.cgi?id=211718
+
+ Reviewed by Eric Carlson.
+
+ * UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp:
+ (WebKit::UserMediaCaptureManagerProxy::SourceProxy::SourceProxy):
+ (WebKit::UserMediaCaptureManagerProxy::SourceProxy::~SourceProxy):
+ * WebProcess/GPU/webrtc/MediaRecorderPrivate.cpp:
+ (WebKit::MediaRecorderPrivate::MediaRecorderPrivate):
+ (WebKit::MediaRecorderPrivate::~MediaRecorderPrivate):
+ (WebKit::MediaRecorderPrivate::videoSampleAvailable):
+ (WebKit::MediaRecorderPrivate::stopRecording):
+ (WebKit::MediaRecorderPrivate::sampleBufferUpdated): Deleted.
+ * WebProcess/GPU/webrtc/MediaRecorderPrivate.h:
+
2020-05-12 Mark Lam <[email protected]>
Wasm::enableFastMemory() was called too late.
Modified: trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp (261552 => 261553)
--- trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -50,8 +50,9 @@
using namespace WebCore;
class UserMediaCaptureManagerProxy::SourceProxy
- : public RealtimeMediaSource::Observer
- , public RealtimeMediaSource::AudioSampleObserver
+ : private RealtimeMediaSource::Observer
+ , private RealtimeMediaSource::AudioSampleObserver
+ , private RealtimeMediaSource::VideoSampleObserver
, public SharedRingBufferStorage::Client {
WTF_MAKE_FAST_ALLOCATED;
public:
@@ -62,13 +63,32 @@
, m_ringBuffer(makeUniqueRef<SharedRingBufferStorage>(makeUniqueRef<SharedRingBufferStorage>(this)))
{
m_source->addObserver(*this);
- m_source->addAudioSampleObserver(*this);
+ switch (m_source->type()) {
+ case RealtimeMediaSource::Type::Audio:
+ m_source->addAudioSampleObserver(*this);
+ break;
+ case RealtimeMediaSource::Type::Video:
+ m_source->addVideoSampleObserver(*this);
+ break;
+ case RealtimeMediaSource::Type::None:
+ ASSERT_NOT_REACHED();
+ }
}
~SourceProxy()
{
storage().invalidate();
- m_source->removeAudioSampleObserver(*this);
+
+ switch (m_source->type()) {
+ case RealtimeMediaSource::Type::Audio:
+ m_source->removeAudioSampleObserver(*this);
+ break;
+ case RealtimeMediaSource::Type::Video:
+ m_source->removeVideoSampleObserver(*this);
+ break;
+ case RealtimeMediaSource::Type::None:
+ ASSERT_NOT_REACHED();
+ }
m_source->removeObserver(*this);
}
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.cpp (261552 => 261553)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.cpp 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.cpp 2020-05-12 11:45:25 UTC (rev 261553)
@@ -64,7 +64,7 @@
width = selectedTracks.videoTrack->settings().width();
}
- m_connection->sendWithAsyncReply(Messages::RemoteMediaRecorderManager::CreateRecorder { m_identifier, !!selectedTracks.audioTrack, width, height }, [this, weakThis = makeWeakPtr(this), audioTrack = makeRefPtr(selectedTracks.audioTrack)](auto&& exception) {
+ m_connection->sendWithAsyncReply(Messages::RemoteMediaRecorderManager::CreateRecorder { m_identifier, !!selectedTracks.audioTrack, width, height }, [this, weakThis = makeWeakPtr(this), audioTrack = makeRefPtr(selectedTracks.audioTrack), videoTrack = makeRefPtr(selectedTracks.videoTrack)](auto&& exception) {
if (!weakThis)
return;
if (exception)
@@ -71,6 +71,8 @@
return m_errorCallback(Exception { exception->code, WTFMove(exception->message) });
if (audioTrack)
setAudioSource(&audioTrack->source());
+ if (videoTrack)
+ setVideoSource(&videoTrack->source());
}, 0);
}
@@ -77,13 +79,12 @@
MediaRecorderPrivate::~MediaRecorderPrivate()
{
setAudioSource(nullptr);
+ setVideoSource(nullptr);
m_connection->send(Messages::RemoteMediaRecorderManager::ReleaseRecorder { m_identifier }, 0);
}
-void MediaRecorderPrivate::sampleBufferUpdated(const WebCore::MediaStreamTrackPrivate& track, WebCore::MediaSample& sample)
+void MediaRecorderPrivate::videoSampleAvailable(MediaSample& sample)
{
- if (track.id() != m_recordedVideoTrackID)
- return;
if (auto remoteSample = RemoteVideoSample::create(sample))
m_connection->send(Messages::RemoteMediaRecorder::VideoSampleAvailable { WTFMove(*remoteSample) }, m_identifier);
}
@@ -128,6 +129,7 @@
void MediaRecorderPrivate::stopRecording()
{
setAudioSource(nullptr);
+ setVideoSource(nullptr);
m_connection->send(Messages::RemoteMediaRecorder::StopRecording { }, m_identifier);
}
Modified: trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.h (261552 => 261553)
--- trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.h 2020-05-12 11:17:57 UTC (rev 261552)
+++ trunk/Source/WebKit/WebProcess/GPU/webrtc/MediaRecorderPrivate.h 2020-05-12 11:45:25 UTC (rev 261553)
@@ -55,7 +55,7 @@
private:
// WebCore::MediaRecorderPrivate
- void sampleBufferUpdated(const WebCore::MediaStreamTrackPrivate&, WebCore::MediaSample&) final;
+ void videoSampleAvailable(WebCore::MediaSample&) final;
void fetchData(CompletionHandler<void(RefPtr<WebCore::SharedBuffer>&&, const String& mimeType)>&&) final;
void stopRecording() final;
void audioSamplesAvailable(const WTF::MediaTime&, const WebCore::PlatformAudioData&, const WebCore::AudioStreamDescription&, size_t) final;
