Diff
Modified: trunk/LayoutTests/ChangeLog (233603 => 233604)
--- trunk/LayoutTests/ChangeLog 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/LayoutTests/ChangeLog 2018-07-06 23:47:00 UTC (rev 233604)
@@ -1,3 +1,16 @@
+2018-07-06 Youenn Fablet <[email protected]>
+
+ WebRTC MediaStreamTrack Enable / Disable causes video delay / lag
+ https://bugs.webkit.org/show_bug.cgi?id=186889
+ <rdar://problem/41370285>
+
+ Reviewed by Eric Carlson.
+
+ Test is no longer valid since we are now sending 0 bytes for audio tracks.
+
+ * webrtc/audio-muted-stats2-expected.txt: Removed.
+ * webrtc/audio-muted-stats2.html: Removed.
+
2018-07-06 Ryan Haddad <[email protected]>
Skip imported/w3c/web-platform-tests/infrastructure/assumptions/html-elements.html on debug.
Deleted: trunk/LayoutTests/webrtc/audio-muted-stats2-expected.txt (233603 => 233604)
--- trunk/LayoutTests/webrtc/audio-muted-stats2-expected.txt 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/LayoutTests/webrtc/audio-muted-stats2-expected.txt 2018-07-06 23:47:00 UTC (rev 233604)
@@ -1,3 +0,0 @@
-
-PASS Audio silent data not being sent in case track is muted from the start
-
Deleted: trunk/LayoutTests/webrtc/audio-muted-stats2.html (233603 => 233604)
--- trunk/LayoutTests/webrtc/audio-muted-stats2.html 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/LayoutTests/webrtc/audio-muted-stats2.html 2018-07-06 23:47:00 UTC (rev 233604)
@@ -1,64 +0,0 @@
-<!doctype html>
-<html>
- <head>
- <meta charset="utf-8">
- <title>Testing basic video exchange from offerer to receiver</title>
- <script src=""
- <script src=""
- </head>
- <body>
- <script src =""
- <script>
-function getOutboundRTPStats(connection)
-{
- return connection.getStats().then((report) => {
- var stats;
- report.forEach((statItem) => {
- if (statItem.type === "outbound-rtp") {
- stats = statItem;
- }
- });
- return stats;
- });
-}
-
-function checkOutboundBytesSentNotIncreasing(firstConnection, statsFirstConnection, count)
-{
- return getOutboundRTPStats(firstConnection).then((stats) => {
- if (stats.bytesSent > statsFirstConnection.bytesSent)
- return Promise.reject("outbound stats bytes sent increasing");
- if (++count === 10)
- return;
- return waitFor(50).then(() => {
- return checkOutboundBytesSentNotIncreasing(firstConnection, statsFirstConnection, count);
- });
- });
-}
-
-var track, firstConnection;
-promise_test((test) => {
- if (window.testRunner)
- testRunner.setUserMediaPermission(true);
-
- return navigator.mediaDevices.getUserMedia({ audio: true}).then((stream) => {
- track = stream.getAudioTracks()[0];
- track.enabled = false;
- return new Promise((resolve, reject) => {
- createConnections((connection) => {
- firstConnection = connection;
- firstConnection.addTrack(track, stream);
- }, (connection) => {
- connection._ontrack_ = resolve;
- });
- setTimeout(() => reject("Test timed out"), 5000);
- });
- }).then(() => {
- return getOutboundRTPStats(firstConnection);
- }).then((stats) => {
- statsFirstConnection = stats;
- return checkOutboundBytesSentNotIncreasing(firstConnection, statsFirstConnection, 0);
- });
-}, "Audio silent data not being sent in case track is muted from the start");
- </script>
- </body>
-</html>
Modified: trunk/Source/WebCore/ChangeLog (233603 => 233604)
--- trunk/Source/WebCore/ChangeLog 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/Source/WebCore/ChangeLog 2018-07-06 23:47:00 UTC (rev 233604)
@@ -1,5 +1,49 @@
2018-07-06 Youenn Fablet <[email protected]>
+ WebRTC MediaStreamTrack Enable / Disable causes video delay / lag
+ https://bugs.webkit.org/show_bug.cgi?id=186889
+ <rdar://problem/41370285>
+
+ Reviewed by Eric Carlson.
+
+ Libwebrtc expects a continuous flow of calls for audio data since the API
+ does not provide any possiblity to give timestamps.
+
+ We were optimizing previously when a source is muted so that we would not transmit audio data.
+ This breaks synchronization between audio and video frames (which are timestamped).
+
+ This patch reverts the optimization and instead makes sure to send zeros for silenced audio tracks.
+
+ This requires MediaStreamTrackPrivate to send audio data even if disabled,
+ so that RealtimeOutgoingAudioSource will continue sending zeros at the correct pace.
+ This also requires WebAudioSourceProviderAVFObjC to exit early if its track is disabled.
+
+ Covered by existing tests.
+ Manual testing shows that synchronization is kept.
+
+ * platform/mediastream/MediaStreamTrackPrivate.cpp:
+ (WebCore::MediaStreamTrackPrivate::audioSamplesAvailable):
+ * platform/mediastream/RealtimeOutgoingAudioSource.cpp:
+ (WebCore::RealtimeOutgoingAudioSource::RealtimeOutgoingAudioSource):
+ (WebCore::RealtimeOutgoingAudioSource::initializeConverter):
+ (WebCore::RealtimeOutgoingAudioSource::stop):
+ (WebCore::RealtimeOutgoingAudioSource::sourceMutedChanged):
+ (WebCore::RealtimeOutgoingAudioSource::sourceEnabledChanged):
+ (WebCore::RealtimeOutgoingAudioSource::handleMutedIfNeeded): Deleted.
+ * platform/mediastream/RealtimeOutgoingAudioSource.h:
+ (WebCore::RealtimeOutgoingAudioSource::pullAudioData):
+ (WebCore::RealtimeOutgoingAudioSource::isSilenced const):
+ (WebCore::RealtimeOutgoingAudioSource::sendSilence): Deleted.
+ * platform/mediastream/mac/RealtimeOutgoingAudioSourceCocoa.cpp:
+ (WebCore::RealtimeOutgoingAudioSourceCocoa::pullAudioData):
+ (WebCore::RealtimeOutgoingAudioSourceCocoa::handleMutedIfNeeded): Deleted.
+ (WebCore::RealtimeOutgoingAudioSourceCocoa::sendSilence): Deleted.
+ * platform/mediastream/mac/RealtimeOutgoingAudioSourceCocoa.h:
+ * platform/mediastream/mac/WebAudioSourceProviderAVFObjC.mm:
+ (WebCore::WebAudioSourceProviderAVFObjC::audioSamplesAvailable):
+
+2018-07-06 Youenn Fablet <[email protected]>
+
Make RealtimeOutgoingVideoSource use DestructionThread::Main
https://bugs.webkit.org/show_bug.cgi?id=187402
Modified: trunk/Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.cpp (233603 => 233604)
--- trunk/Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.cpp 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.cpp 2018-07-06 23:47:00 UTC (rev 233604)
@@ -216,9 +216,6 @@
updateReadyState();
}
- if (!enabled())
- return;
-
for (auto& observer : m_observers)
observer->audioSamplesAvailable(*this, mediaTime, data, description, sampleCount);
}
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingAudioSource.cpp (233603 => 233604)
--- trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingAudioSource.cpp 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingAudioSource.cpp 2018-07-06 23:47:00 UTC (rev 233604)
@@ -38,7 +38,6 @@
RealtimeOutgoingAudioSource::RealtimeOutgoingAudioSource(Ref<MediaStreamTrackPrivate>&& audioSource)
: m_audioSource(WTFMove(audioSource))
- , m_silenceAudioTimer(*this, &RealtimeOutgoingAudioSource::sendSilence)
{
m_audioSource->addObserver(*this);
initializeConverter();
@@ -58,13 +57,11 @@
{
m_muted = m_audioSource->muted();
m_enabled = m_audioSource->enabled();
- handleMutedIfNeeded();
}
void RealtimeOutgoingAudioSource::stop()
{
ASSERT(isMainThread());
- m_silenceAudioTimer.stop();
m_audioSource->removeObserver(*this);
}
@@ -71,24 +68,13 @@
void RealtimeOutgoingAudioSource::sourceMutedChanged()
{
m_muted = m_audioSource->muted();
- handleMutedIfNeeded();
}
void RealtimeOutgoingAudioSource::sourceEnabledChanged()
{
m_enabled = m_audioSource->enabled();
- handleMutedIfNeeded();
}
-void RealtimeOutgoingAudioSource::handleMutedIfNeeded()
-{
- bool isSilenced = m_muted || !m_enabled;
- if (isSilenced && !m_silenceAudioTimer.isActive())
- m_silenceAudioTimer.startRepeating(1_s);
- if (!isSilenced && m_silenceAudioTimer.isActive())
- m_silenceAudioTimer.stop();
-}
-
} // namespace WebCore
#endif // USE(LIBWEBRTC)
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingAudioSource.h (233603 => 233604)
--- trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingAudioSource.h 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeOutgoingAudioSource.h 2018-07-06 23:47:00 UTC (rev 233604)
@@ -58,13 +58,11 @@
protected:
explicit RealtimeOutgoingAudioSource(Ref<MediaStreamTrackPrivate>&&);
- virtual void handleMutedIfNeeded();
- virtual void sendSilence() { };
- virtual void pullAudioData() { };
+ virtual void pullAudioData() { }
+ bool isSilenced() const { return m_muted || !m_enabled; }
+
Vector<webrtc::AudioTrackSinkInterface*> m_sinks;
- bool m_muted { false };
- bool m_enabled { true };
private:
virtual void AddSink(webrtc::AudioTrackSinkInterface* sink) { m_sinks.append(sink); }
@@ -102,8 +100,8 @@
void initializeConverter();
Ref<MediaStreamTrackPrivate> m_audioSource;
-
- Timer m_silenceAudioTimer;
+ bool m_muted { false };
+ bool m_enabled { true };
};
} // namespace WebCore
Modified: trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.cpp (233603 => 233604)
--- trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.cpp 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.cpp 2018-07-06 23:47:00 UTC (rev 233604)
@@ -43,14 +43,6 @@
{
}
-void RealtimeOutgoingAudioSourceLibWebRTC::handleMutedIfNeeded()
-{
-}
-
-void RealtimeOutgoingAudioSourceLibWebRTC::sendSilence()
-{
-}
-
void RealtimeOutgoingAudioSourceLibWebRTC::pullAudioData()
{
}
Modified: trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.h (233603 => 233604)
--- trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.h 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.h 2018-07-06 23:47:00 UTC (rev 233604)
@@ -43,9 +43,6 @@
bool hasBufferedEnoughData() final;
void pullAudioData() final;
-
- void handleMutedIfNeeded() final;
- void sendSilence() final;
};
} // namespace WebCore
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingAudioSourceCocoa.cpp (233603 => 233604)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingAudioSourceCocoa.cpp 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingAudioSourceCocoa.cpp 2018-07-06 23:47:00 UTC (rev 233604)
@@ -54,30 +54,6 @@
return RealtimeOutgoingAudioSourceCocoa::create(WTFMove(audioSource));
}
-void RealtimeOutgoingAudioSourceCocoa::handleMutedIfNeeded()
-{
- bool isSilenced = m_muted || !m_enabled;
- m_sampleConverter->setMuted(isSilenced);
-
- RealtimeOutgoingAudioSource::handleMutedIfNeeded();
-}
-
-void RealtimeOutgoingAudioSourceCocoa::sendSilence()
-{
- LibWebRTCProvider::callOnWebRTCSignalingThread([this, protectedThis = makeRef(*this)] {
- size_t chunkSampleCount = m_outputStreamDescription.sampleRate() / 100;
- size_t bufferSize = chunkSampleCount * LibWebRTCAudioFormat::sampleByteSize * m_outputStreamDescription.numberOfChannels();
-
- if (!bufferSize)
- return;
-
- m_audioBuffer.grow(bufferSize);
- memset(m_audioBuffer.data(), 0, bufferSize);
- for (auto sink : m_sinks)
- sink->OnData(m_audioBuffer.data(), LibWebRTCAudioFormat::sampleSize, m_outputStreamDescription.sampleRate(), m_outputStreamDescription.numberOfChannels(), chunkSampleCount);
- });
-}
-
bool RealtimeOutgoingAudioSourceCocoa::isReachingBufferedAudioDataHighLimit()
{
auto writtenAudioDuration = m_writeCount / m_inputStreamDescription.sampleRate();
@@ -153,6 +129,9 @@
bufferList.mBuffers[0].mDataByteSize = bufferSize;
bufferList.mBuffers[0].mData = m_audioBuffer.data();
+ if (isSilenced() != m_sampleConverter->muted())
+ m_sampleConverter->setMuted(isSilenced());
+
m_sampleConverter->pullAvalaibleSamplesAsChunks(bufferList, chunkSampleCount, m_readCount, [this, chunkSampleCount] {
m_readCount += chunkSampleCount;
for (auto sink : m_sinks)
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingAudioSourceCocoa.h (233603 => 233604)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingAudioSourceCocoa.h 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingAudioSourceCocoa.h 2018-07-06 23:47:00 UTC (rev 233604)
@@ -52,9 +52,6 @@
void pullAudioData() final;
- void handleMutedIfNeeded() final;
- void sendSilence() final;
-
Ref<AudioSampleDataSource> m_sampleConverter;
CAAudioStreamDescription m_inputStreamDescription;
CAAudioStreamDescription m_outputStreamDescription;
Modified: trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderAVFObjC.mm (233603 => 233604)
--- trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderAVFObjC.mm 2018-07-06 23:40:07 UTC (rev 233603)
+++ trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderAVFObjC.mm 2018-07-06 23:47:00 UTC (rev 233604)
@@ -159,8 +159,11 @@
}
}
-void WebAudioSourceProviderAVFObjC::audioSamplesAvailable(MediaStreamTrackPrivate&, const MediaTime&, const PlatformAudioData& data, const AudioStreamDescription& description, size_t frameCount)
+void WebAudioSourceProviderAVFObjC::audioSamplesAvailable(MediaStreamTrackPrivate& track, const MediaTime&, const PlatformAudioData& data, const AudioStreamDescription& description, size_t frameCount)
{
+ if (!track.enabled())
+ return;
+
ASSERT(description.platformDescription().type == PlatformDescription::CAAudioStreamBasicType);
auto& basicDescription = *WTF::get<const AudioStreamBasicDescription*>(description.platformDescription().description);
if (!m_inputDescription || m_inputDescription->streamDescription() != basicDescription)
