Diff
Modified: trunk/Source/WebCore/ChangeLog (276242 => 276243)
--- trunk/Source/WebCore/ChangeLog 2021-04-19 09:49:20 UTC (rev 276242)
+++ trunk/Source/WebCore/ChangeLog 2021-04-19 09:53:37 UTC (rev 276243)
@@ -1,3 +1,44 @@
+2021-04-19 Youenn Fablet <[email protected]>
+
+ Reduce crackling at start of playing a live audio track
+ https://bugs.webkit.org/show_bug.cgi?id=218898
+ <rdar://problem/71625010>
+
+ Reviewed by Eric Carlson.
+
+ We often hear crackles at the beginning of playing a MediaStreamTrack.
+ This is due to starting at a point where there is not enough data, so we start rendering and quickly run short of data.
+ Our heuristic to start was not good since we were not accounting to LibWebRTC audio module which sends audio samples by batch of 5 samples.
+ So we would have needed to at least have 6 samples to be sure to not run out of data.
+
+ What this patch does:
+ - Add an extra parameter to AudioSampleDataSource so that we do not start until we have at least been pushed a given number of sample count.
+ - Change LibWebRTAudioModule to send audio samples by batch of 3 (hence 30 ms) instead of 5
+ - Set this new parameter to 2 for local tracks and 4 for remote tracks
+ - Add a new boolean to AudioSampleDataSource to know whether we start the track for the first time or not. If we start it, use the new parameter to buffer enough data.
+ Otherwise, use current heuristic.
+
+ We also reduce the AudioSampleDataSource buffer to 0.5 seconds instead of 2 seconds, since 2 seconds is too much for real time audio.
+
+ Manually tested.
+
+ * platform/audio/cocoa/AudioSampleDataSource.h:
+ * platform/audio/cocoa/AudioSampleDataSource.mm:
+ (WebCore::AudioSampleDataSource::create):
+ (WebCore::AudioSampleDataSource::AudioSampleDataSource):
+ (WebCore::AudioSampleDataSource::pullSamplesInternal):
+ * platform/mediastream/libwebrtc/LibWebRTCAudioModule.cpp:
+ (WebCore::LibWebRTCAudioModule::pollFromSource):
+ * platform/mediastream/libwebrtc/LibWebRTCAudioModule.h:
+ * platform/mediastream/mac/AudioMediaStreamTrackRendererCocoa.cpp:
+ (WebCore::pollSamplesCount):
+ (WebCore::AudioMediaStreamTrackRendererCocoa::pushSamples):
+ * platform/mediastream/mac/MediaStreamTrackAudioSourceProviderCocoa.cpp:
+ (WebCore::MediaStreamTrackAudioSourceProviderCocoa::MediaStreamTrackAudioSourceProviderCocoa):
+ * platform/mediastream/mac/WebAudioSourceProviderCocoa.h:
+ * platform/mediastream/mac/WebAudioSourceProviderCocoa.mm:
+ (WebCore::WebAudioSourceProviderCocoa::prepare):
+
2021-04-19 Imanol Fernandez <[email protected]>
Enable GL_ANGLE_instanced_arrays WebGL extension in WPE
Modified: trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.h (276242 => 276243)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.h 2021-04-19 09:49:20 UTC (rev 276242)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.h 2021-04-19 09:53:37 UTC (rev 276243)
@@ -47,7 +47,7 @@
#endif
{
public:
- static Ref<AudioSampleDataSource> create(size_t, WTF::LoggerHelper&);
+ static Ref<AudioSampleDataSource> create(size_t, WTF::LoggerHelper&, size_t waitToStartForPushCount = 2);
~AudioSampleDataSource();
@@ -80,7 +80,7 @@
static constexpr float EquivalentToMaxVolume = 0.95;
private:
- AudioSampleDataSource(size_t, LoggerHelper&);
+ AudioSampleDataSource(size_t, LoggerHelper&, size_t waitToStartForPushCount);
OSStatus setupConverter();
bool pullSamplesInternal(AudioBufferList&, size_t, uint64_t, double, PullMode);
@@ -98,7 +98,9 @@
#endif
uint64_t m_lastPushedSampleCount { 0 };
+ size_t m_waitToStartForPushCount { 2 };
MediaTime m_expectedNextPushedSampleTime { MediaTime::invalidTime() };
+ bool m_isFirstPull { true };
MediaTime m_inputSampleOffset;
int64_t m_outputSampleOffset { 0 };
Modified: trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm (276242 => 276243)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm 2021-04-19 09:49:20 UTC (rev 276242)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm 2021-04-19 09:53:37 UTC (rev 276243)
@@ -45,13 +45,14 @@
using namespace PAL;
using namespace JSC;
-Ref<AudioSampleDataSource> AudioSampleDataSource::create(size_t maximumSampleCount, LoggerHelper& loggerHelper)
+Ref<AudioSampleDataSource> AudioSampleDataSource::create(size_t maximumSampleCount, LoggerHelper& loggerHelper, size_t waitToStartForPushCount)
{
- return adoptRef(*new AudioSampleDataSource(maximumSampleCount, loggerHelper));
+ return adoptRef(*new AudioSampleDataSource(maximumSampleCount, loggerHelper, waitToStartForPushCount));
}
-AudioSampleDataSource::AudioSampleDataSource(size_t maximumSampleCount, LoggerHelper& loggerHelper)
- : m_inputSampleOffset(MediaTime::invalidTime())
+AudioSampleDataSource::AudioSampleDataSource(size_t maximumSampleCount, LoggerHelper& loggerHelper, size_t waitToStartForPushCount)
+ : m_waitToStartForPushCount(waitToStartForPushCount)
+ , m_inputSampleOffset(MediaTime::invalidTime())
, m_ringBuffer(makeUniqueRef<CARingBuffer>())
, m_maximumSampleCount(maximumSampleCount)
#if !RELEASE_LOG_DISABLED
@@ -223,16 +224,28 @@
if (m_shouldComputeOutputSampleOffset) {
uint64_t buffered = endFrame - startFrame;
- if (buffered < sampleCount * 2 || (m_endFrameWhenNotEnoughData && m_endFrameWhenNotEnoughData == endFrame)) {
- AudioSampleBufferList::zeroABL(buffer, byteCount);
- return false;
- }
+ if (m_isFirstPull) {
+ if (buffered >= m_waitToStartForPushCount * m_lastPushedSampleCount) {
+ m_outputSampleOffset = startFrame - timeStamp;
+ m_shouldComputeOutputSampleOffset = false;
+ m_endFrameWhenNotEnoughData = 0;
+ } else {
+ // We wait for one chunk of value before starting to play.
+ AudioSampleBufferList::zeroABL(buffer, byteCount);
+ return false;
+ }
+ } else {
+ if (buffered < sampleCount * 2 || (m_endFrameWhenNotEnoughData && m_endFrameWhenNotEnoughData == endFrame)) {
+ AudioSampleBufferList::zeroABL(buffer, byteCount);
+ return false;
+ }
- m_shouldComputeOutputSampleOffset = false;
- m_endFrameWhenNotEnoughData = 0;
+ m_shouldComputeOutputSampleOffset = false;
+ m_endFrameWhenNotEnoughData = 0;
- m_outputSampleOffset = (endFrame - sampleCount) - timeStamp;
- m_outputSampleOffset -= computeOffsetDelay(m_outputDescription->sampleRate(), m_lastPushedSampleCount);
+ m_outputSampleOffset = (endFrame - sampleCount) - timeStamp;
+ m_outputSampleOffset -= computeOffsetDelay(m_outputDescription->sampleRate(), m_lastPushedSampleCount);
+ }
}
timeStamp += m_outputSampleOffset;
@@ -252,6 +265,8 @@
return false;
}
+ m_isFirstPull = false;
+
if (mode == Copy) {
m_ringBuffer->fetch(&buffer, sampleCount, timeStamp, CARingBuffer::Copy);
if (m_volume < EquivalentToMaxVolume)
Modified: trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.cpp (276242 => 276243)
--- trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.cpp 2021-04-19 09:49:20 UTC (rev 276242)
+++ trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.cpp 2021-04-19 09:53:37 UTC (rev 276243)
@@ -71,8 +71,7 @@
// libwebrtc uses 10ms frames.
const unsigned frameLengthMs = 1000 * LibWebRTCAudioFormat::chunkSampleCount / LibWebRTCAudioFormat::sampleRate;
-const unsigned pollSamples = 5;
-const unsigned pollInterval = 5 * frameLengthMs;
+const unsigned pollInterval = LibWebRTCAudioModule::PollSamplesCount * frameLengthMs;
const unsigned channels = 2;
void LibWebRTCAudioModule::pollAudioData()
@@ -101,7 +100,7 @@
if (!m_audioTransport)
return;
- for (unsigned i = 0; i < pollSamples; i++) {
+ for (unsigned i = 0; i < PollSamplesCount; i++) {
int64_t elapsedTime = -1;
int64_t ntpTime = -1;
char data[LibWebRTCAudioFormat::sampleByteSize * channels * LibWebRTCAudioFormat::chunkSampleCount];
Modified: trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.h (276242 => 276243)
--- trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.h 2021-04-19 09:49:20 UTC (rev 276242)
+++ trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.h 2021-04-19 09:53:37 UTC (rev 276243)
@@ -45,6 +45,8 @@
public:
LibWebRTCAudioModule();
+ static constexpr unsigned PollSamplesCount = 3;
+
private:
template<typename U> U shouldNotBeCalled(U value) const
{
Modified: trunk/Source/WebCore/platform/mediastream/mac/AudioMediaStreamTrackRendererCocoa.cpp (276242 => 276243)
--- trunk/Source/WebCore/platform/mediastream/mac/AudioMediaStreamTrackRendererCocoa.cpp 2021-04-19 09:49:20 UTC (rev 276242)
+++ trunk/Source/WebCore/platform/mediastream/mac/AudioMediaStreamTrackRendererCocoa.cpp 2021-04-19 09:53:37 UTC (rev 276243)
@@ -31,6 +31,7 @@
#include "AudioMediaStreamTrackRendererUnit.h"
#include "AudioSampleDataSource.h"
#include "CAAudioStreamDescription.h"
+#include "LibWebRTCAudioModule.h"
namespace WebCore {
@@ -74,12 +75,23 @@
m_shouldReset = true;
}
+static unsigned pollSamplesCount()
+{
+#if USE(LIBWEBRTC)
+ return LibWebRTCAudioModule::PollSamplesCount + 1;
+#else
+ return 2;
+#endif
+}
+
void AudioMediaStreamTrackRendererCocoa::pushSamples(const MediaTime& sampleTime, const PlatformAudioData& audioData, const AudioStreamDescription& description, size_t sampleCount)
{
ASSERT(!isMainThread());
ASSERT(description.platformDescription().type == PlatformDescription::CAAudioStreamBasicType);
if (!m_dataSource || m_shouldReset || !m_dataSource->inputDescription() || *m_dataSource->inputDescription() != description) {
- auto dataSource = AudioSampleDataSource::create(description.sampleRate() * 2, *this);
+ // FIXME: For non libwebrtc sources, we can probably reduce poll samples count to 2.
+
+ auto dataSource = AudioSampleDataSource::create(description.sampleRate() * 0.5, *this, pollSamplesCount());
if (dataSource->setInputFormat(toCAAudioStreamDescription(description))) {
ERROR_LOG(LOGIDENTIFIER, "Unable to set the input format of data source");
Modified: trunk/Source/WebCore/platform/mediastream/mac/MediaStreamTrackAudioSourceProviderCocoa.cpp (276242 => 276243)
--- trunk/Source/WebCore/platform/mediastream/mac/MediaStreamTrackAudioSourceProviderCocoa.cpp 2021-04-19 09:49:20 UTC (rev 276242)
+++ trunk/Source/WebCore/platform/mediastream/mac/MediaStreamTrackAudioSourceProviderCocoa.cpp 2021-04-19 09:53:37 UTC (rev 276243)
@@ -28,6 +28,8 @@
#if ENABLE(WEB_AUDIO) && ENABLE(MEDIA_STREAM)
+#import "LibWebRTCAudioModule.h"
+
namespace WebCore {
Ref<MediaStreamTrackAudioSourceProviderCocoa> MediaStreamTrackAudioSourceProviderCocoa::create(MediaStreamTrackPrivate& source)
@@ -39,6 +41,10 @@
: m_captureSource(makeWeakPtr(source))
, m_source(source.source())
{
+#if USE(LIBWEBRTC)
+ if (m_source->isIncomingAudioSource())
+ setPollSamplesCount(LibWebRTCAudioModule::PollSamplesCount + 1);
+#endif
}
MediaStreamTrackAudioSourceProviderCocoa::~MediaStreamTrackAudioSourceProviderCocoa()
Modified: trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderCocoa.h (276242 => 276243)
--- trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderCocoa.h 2021-04-19 09:49:20 UTC (rev 276242)
+++ trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderCocoa.h 2021-04-19 09:53:37 UTC (rev 276243)
@@ -60,6 +60,8 @@
protected:
void receivedNewAudioSamples(const PlatformAudioData&, const AudioStreamDescription&, size_t);
+ void setPollSamplesCount(size_t);
+
private:
virtual void hasNewClient(AudioSourceProviderClient*) = 0;
#if !RELEASE_LOG_DISABLED
@@ -80,10 +82,16 @@
std::unique_ptr<WebAudioBufferList> m_audioBufferList;
RefPtr<AudioSampleDataSource> m_dataSource;
+ size_t m_pollSamplesCount { 3 };
uint64_t m_writeCount { 0 };
uint64_t m_readCount { 0 };
};
+inline void WebAudioSourceProviderCocoa::setPollSamplesCount(size_t count)
+{
+ m_pollSamplesCount = count;
}
+}
+
#endif // ENABLE(WEB_AUDIO)
Modified: trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderCocoa.mm (276242 => 276243)
--- trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderCocoa.mm 2021-04-19 09:49:20 UTC (rev 276242)
+++ trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderCocoa.mm 2021-04-19 09:53:37 UTC (rev 276243)
@@ -120,7 +120,7 @@
m_audioBufferList = makeUnique<WebAudioBufferList>(m_outputDescription.value());
if (!m_dataSource)
- m_dataSource = AudioSampleDataSource::create(kRingBufferDuration * sampleRate, loggerHelper());
+ m_dataSource = AudioSampleDataSource::create(kRingBufferDuration * sampleRate, loggerHelper(), m_pollSamplesCount);
m_dataSource->setInputFormat(m_inputDescription.value());
m_dataSource->setOutputFormat(m_outputDescription.value());
