Diff
Modified: trunk/Source/WebCore/ChangeLog (287156 => 287157)
--- trunk/Source/WebCore/ChangeLog 2021-12-16 22:00:44 UTC (rev 287156)
+++ trunk/Source/WebCore/ChangeLog 2021-12-16 22:02:54 UTC (rev 287157)
@@ -1,3 +1,45 @@
+2021-12-16 Youenn Fablet <[email protected]>
+
+ Allow AudioSampleDataSource to increase/decrease buffered data progressively
+ https://bugs.webkit.org/show_bug.cgi?id=233422
+
+ Reviewed by Eric Carlson.
+
+ AudioSampleDataSource does the link between push audio sources and pull audio sinks.
+ As such, it needs to do buffering. If buffering is too small, data may be missing and audio glitches will be heard.
+ If buffering is too large, latency will be added which might be undesirable, especially if audio is being played with video.
+ We generally want buffered audio to stay within a certain range.
+
+ To make this happen, when buffering is too high, we convert the data with a slightly lower sample rate to push less samples, until we are back to normal buffering.
+ Conversely, when buffering is too low, we convert the data with a slightly higher sample rate to push more samples, until we are back to normal buffering.
+ We do this with 3 converters that we select based on amount of buffered data.
+ This behavior is encapsulated in AudioSampleDataConverter.
+
+ We simplify AudioSampleDataSource implementation by always recomputing the sample offset when there is not enough data.
+ In that case, we wait for 50ms of buffered data, which is the average buffer we expect, to restart pulling data.
+ All values owned by AudioSampleDataSource (m_expectedNextPushedSampleTimeValue, m_converterInputOffset, m_converterInputOffset, m_outputSampleOffset, m_lastBufferedAmount)
+ are all in the outgoing timeline/sampleRate.
+
+ This adaptation is only enabled when AudioSampleDataSource::pullSamples is called.
+ For pullAvailableSamplesAsChunks and pullAvailableSampleChunk, the puller is supposed to be in sync with the pusher.
+ For that reason, we make sure to always write the expected number of audio frames when pullSamples is called, even if converter fails.
+
+ We fix a potential busy loop in AudioSampleDataSource::pullAvailableSamplesAsChunks in case endFrame is lower than startFrame, which is computed from timeStamp input parameter.
+
+ Update MockAudioSharedUnit to be closer to a real source by increasing the queue priority and schedule rendering tasks from the queue instead of relying on
+ a main thread timer which can have hiccups.
+
+ Manually tested.
+
+ * SourcesCocoa.txt:
+ * WebCore.xcodeproj/project.pbxproj:
+ * platform/audio/cocoa/AudioSampleDataConverter.h: Added.
+ * platform/audio/cocoa/AudioSampleDataConverter.mm: Added.
+ * platform/audio/cocoa/AudioSampleDataSource.h:
+ * platform/audio/cocoa/AudioSampleDataSource.mm:
+ * platform/mediastream/mac/MockAudioSharedUnit.h:
+ * platform/mediastream/mac/MockAudioSharedUnit.mm:
+
2021-12-16 Sihui Liu <[email protected]>
REGRESSION (r286601): storage/filesystemaccess/sync-access-handle-read-write-worker.html and file-system-access/sandboxed_FileSystemSyncAccessHandle-truncate.https.tentative.worker.html are consistently failing
Modified: trunk/Source/WebCore/SourcesCocoa.txt (287156 => 287157)
--- trunk/Source/WebCore/SourcesCocoa.txt 2021-12-16 22:00:44 UTC (rev 287156)
+++ trunk/Source/WebCore/SourcesCocoa.txt 2021-12-16 22:02:54 UTC (rev 287157)
@@ -224,6 +224,7 @@
platform/audio/cocoa/AudioFileReaderCocoa.cpp
platform/audio/cocoa/AudioOutputUnitAdaptor.cpp
platform/audio/cocoa/AudioSampleBufferList.cpp
+platform/audio/cocoa/AudioSampleDataConverter.mm
platform/audio/cocoa/AudioSampleDataSource.mm
platform/audio/cocoa/CAAudioStreamDescription.cpp
platform/audio/cocoa/CARingBuffer.cpp
@@ -315,7 +316,7 @@
platform/graphics/avfoundation/objc/InbandChapterTrackPrivateAVFObjC.mm @no-unify
platform/graphics/avfoundation/objc/LocalSampleBufferDisplayLayer.mm
platform/graphics/avfoundation/objc/MediaPlaybackTargetPickerMac.mm
-platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm
+platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm @no-unify
platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm @no-unify
platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm
platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm
@@ -322,7 +323,7 @@
platform/graphics/avfoundation/objc/MediaSourcePrivateAVFObjC.mm
platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm @no-unify
platform/graphics/avfoundation/objc/SourceBufferParserAVFObjC.mm
-platform/graphics/avfoundation/objc/VideoLayerManagerObjC.mm
+platform/graphics/avfoundation/objc/VideoLayerManagerObjC.mm @no-unify
platform/graphics/avfoundation/objc/VideoTrackPrivateAVFObjC.cpp
platform/graphics/avfoundation/objc/VideoTrackPrivateMediaSourceAVFObjC.mm
platform/graphics/avfoundation/objc/WebCoreAVFResourceLoader.mm
Modified: trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj (287156 => 287157)
--- trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2021-12-16 22:00:44 UTC (rev 287156)
+++ trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2021-12-16 22:02:54 UTC (rev 287157)
@@ -1228,6 +1228,9 @@
41FCCC3B2746675600892AD6 /* CoreAudioCaptureSource.h in Headers */ = {isa = PBXBuildFile; fileRef = 3F3BB5831E709EE400C701F2 /* CoreAudioCaptureSource.h */; settings = {ATTRIBUTES = (Private, ); }; };
41FCD6B923CE015500C62567 /* SampleBufferDisplayLayer.h in Headers */ = {isa = PBXBuildFile; fileRef = 414598BE23C8AAB8002B9CC8 /* SampleBufferDisplayLayer.h */; settings = {ATTRIBUTES = (Private, ); }; };
41FCD6BB23CE027700C62567 /* LocalSampleBufferDisplayLayer.h in Headers */ = {isa = PBXBuildFile; fileRef = 414598C023C8AD78002B9CC8 /* LocalSampleBufferDisplayLayer.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ 41FFD2C327563E0D00501BBF /* AudioSampleDataConverter.h in Headers */ = {isa = PBXBuildFile; fileRef = 41FFD2C027563DFF00501BBF /* AudioSampleDataConverter.h */; settings = {ATTRIBUTES = (Private, ); }; };
+ 41FFD2C42756570F00501BBF /* VideoLayerManagerObjC.mm in Sources */ = {isa = PBXBuildFile; fileRef = 52D5A18D1C54590300DE34A3 /* VideoLayerManagerObjC.mm */; };
+ 41FFD2C62756573E00501BBF /* MediaPlayerPrivateAVFoundationObjC.mm in Sources */ = {isa = PBXBuildFile; fileRef = DF9AFD7113FC31D80015FEB7 /* MediaPlayerPrivateAVFoundationObjC.mm */; };
427DA71D13735DFA007C57FB /* JSServiceWorkerInternals.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 427DA71B13735DFA007C57FB /* JSServiceWorkerInternals.cpp */; };
427DA71E13735DFA007C57FB /* JSServiceWorkerInternals.h in Headers */ = {isa = PBXBuildFile; fileRef = 427DA71C13735DFA007C57FB /* JSServiceWorkerInternals.h */; };
43107BE218CC19DE00CC18E8 /* SelectorPseudoTypeMap.h in Headers */ = {isa = PBXBuildFile; fileRef = 43107BE118CC19DE00CC18E8 /* SelectorPseudoTypeMap.h */; };
@@ -8943,6 +8946,8 @@
41FCB75F214866FF0038ADC6 /* RTCRtpCodecParameters.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = RTCRtpCodecParameters.h; sourceTree = "<group>"; };
41FCB760214867000038ADC6 /* RTCRtpRtxParameters.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = RTCRtpRtxParameters.h; sourceTree = "<group>"; };
41FCB761214867000038ADC6 /* RTCRtpEncodingParameters.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = RTCRtpEncodingParameters.h; sourceTree = "<group>"; };
+ 41FFD2C027563DFF00501BBF /* AudioSampleDataConverter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AudioSampleDataConverter.h; sourceTree = "<group>"; };
+ 41FFD2C227563E0000501BBF /* AudioSampleDataConverter.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = AudioSampleDataConverter.mm; sourceTree = "<group>"; };
427DA71B13735DFA007C57FB /* JSServiceWorkerInternals.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = JSServiceWorkerInternals.cpp; sourceTree = "<group>"; };
427DA71C13735DFA007C57FB /* JSServiceWorkerInternals.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = JSServiceWorkerInternals.h; sourceTree = "<group>"; };
43107BE118CC19DE00CC18E8 /* SelectorPseudoTypeMap.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SelectorPseudoTypeMap.h; sourceTree = "<group>"; };
@@ -30064,6 +30069,8 @@
1DB66D37253678EA00B671B9 /* AudioOutputUnitAdaptor.h */,
073B87621E43859D0071C0EC /* AudioSampleBufferList.cpp */,
073B87631E43859D0071C0EC /* AudioSampleBufferList.h */,
+ 41FFD2C027563DFF00501BBF /* AudioSampleDataConverter.h */,
+ 41FFD2C227563E0000501BBF /* AudioSampleDataConverter.mm */,
073B87651E43859D0071C0EC /* AudioSampleDataSource.h */,
073B87641E43859D0071C0EC /* AudioSampleDataSource.mm */,
073B87571E40DCFD0071C0EC /* CAAudioStreamDescription.cpp */,
@@ -33322,6 +33329,7 @@
FD31608612B026F700C1A359 /* AudioResampler.h in Headers */,
FD31608812B026F700C1A359 /* AudioResamplerKernel.h in Headers */,
073B87671E4385AC0071C0EC /* AudioSampleBufferList.h in Headers */,
+ 41FFD2C327563E0D00501BBF /* AudioSampleDataConverter.h in Headers */,
073B87691E4385AC0071C0EC /* AudioSampleDataSource.h in Headers */,
FD8C46EC154608E700A5910C /* AudioScheduledSourceNode.h in Headers */,
CDA7982A170A3D0000D45C55 /* AudioSession.h in Headers */,
@@ -38443,6 +38451,7 @@
2D9BF7431DBFDC3E007A7D99 /* MediaKeySystemAccess.cpp in Sources */,
9ACC079825C7267700DC6386 /* MediaKeySystemController.cpp in Sources */,
9ACC079625C725EE00DC6386 /* MediaKeySystemRequest.cpp in Sources */,
+ 41FFD2C62756573E00501BBF /* MediaPlayerPrivateAVFoundationObjC.mm in Sources */,
CDC8B5A2180463470016E685 /* MediaPlayerPrivateMediaSourceAVFObjC.mm in Sources */,
CDA9593524123CB800910EEF /* MediaSessionHelperIOS.mm in Sources */,
07638A9A1884487200E15A1B /* MediaSessionManagerIOS.mm in Sources */,
@@ -39072,6 +39081,7 @@
7CE68344192143A800F4D928 /* UserMessageHandlerDescriptor.cpp in Sources */,
7C73FB07191EF417007DE061 /* UserMessageHandlersNamespace.cpp in Sources */,
3FBC4AF3189881560046EE38 /* VideoFullscreenInterfaceAVKit.mm in Sources */,
+ 41FFD2C42756570F00501BBF /* VideoLayerManagerObjC.mm in Sources */,
26F9A83818A046AC00AEB88A /* ViewportConfiguration.cpp in Sources */,
CDED1C3C24CD305700934E12 /* VP9UtilitiesCocoa.mm in Sources */,
A14832B1187F61E100DA63A6 /* WAKAppKitStubs.m in Sources */,
Modified: trunk/Source/WebCore/platform/audio/cocoa/AudioSampleBufferList.cpp (287156 => 287157)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioSampleBufferList.cpp 2021-12-16 22:00:44 UTC (rev 287156)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioSampleBufferList.cpp 2021-12-16 22:02:54 UTC (rev 287157)
@@ -297,17 +297,12 @@
return 0;
}
- LOG_ERROR("AudioSampleBufferList::copyFrom(%p) AudioConverterFillComplexBuffer returned error %d (%.4s)", this, (int)err, (char*)&err);
+ RELEASE_LOG_ERROR(Media, "AudioSampleBufferList::copyFrom(%p) AudioConverterFillComplexBuffer returned error %d (%.4s)", this, (int)err, (char*)&err);
m_sampleCount = std::min(m_sampleCapacity, static_cast<size_t>(samplesConverted));
zero();
return err;
}
-OSStatus AudioSampleBufferList::copyFrom(AudioSampleBufferList& source, size_t frameCount, AudioConverterRef converter)
-{
- return copyFrom(source.bufferList(), frameCount, converter);
-}
-
OSStatus AudioSampleBufferList::copyFrom(CARingBuffer& ringBuffer, size_t sampleCount, uint64_t startFrame, CARingBuffer::FetchMode mode)
{
reset();
Modified: trunk/Source/WebCore/platform/audio/cocoa/AudioSampleBufferList.h (287156 => 287157)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioSampleBufferList.h 2021-12-16 22:00:44 UTC (rev 287156)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioSampleBufferList.h 2021-12-16 22:02:54 UTC (rev 287157)
@@ -50,7 +50,6 @@
OSStatus copyFrom(const AudioSampleBufferList&, size_t count = SIZE_MAX);
OSStatus copyFrom(const AudioBufferList&, size_t frameCount, AudioConverterRef);
- OSStatus copyFrom(AudioSampleBufferList&, size_t frameCount, AudioConverterRef);
OSStatus copyFrom(CARingBuffer&, size_t frameCount, uint64_t startFrame, CARingBuffer::FetchMode);
OSStatus mixFrom(const AudioSampleBufferList&, size_t count = SIZE_MAX);
Added: trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataConverter.h (0 => 287157)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataConverter.h (rev 0)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataConverter.h 2021-12-16 22:02:54 UTC (rev 287157)
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+typedef struct AudioBufferList AudioBufferList;
+struct AudioStreamBasicDescription;
+typedef struct OpaqueAudioConverter* AudioConverterRef;
+
+namespace WebCore {
+
+class AudioSampleBufferList;
+class CAAudioStreamDescription;
+class PlatformAudioData;
+
+class AudioSampleDataConverter {
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ AudioSampleDataConverter() = default;
+ ~AudioSampleDataConverter();
+
+ OSStatus setFormats(const CAAudioStreamDescription& inputDescription, const CAAudioStreamDescription& outputDescription);
+ bool updateBufferedAmount(size_t currentBufferedAmount);
+ OSStatus convert(const AudioBufferList&, AudioSampleBufferList&, size_t sampleCount);
+ size_t regularBufferSize() const { return m_regularBufferSize; }
+ bool isRegular() const { return m_selectedConverter == m_regularConverter; }
+
+private:
+ size_t m_highBufferSize { 0 };
+ size_t m_regularHighBufferSize { 0 };
+ size_t m_regularBufferSize { 0 };
+ size_t m_regularLowBufferSize { 0 };
+ size_t m_lowBufferSize { 0 };
+
+ class Converter {
+ public:
+ Converter() = default;
+ ~Converter();
+
+ OSStatus initialize(const AudioStreamBasicDescription& inputDescription, const AudioStreamBasicDescription& outputDescription);
+ operator AudioConverterRef() const { return m_audioConverter; }
+
+ private:
+ AudioConverterRef m_audioConverter { nullptr };
+ };
+
+ Converter m_lowConverter;
+ Converter m_regularConverter;
+ Converter m_highConverter;
+ AudioConverterRef m_selectedConverter;
+};
+
+} // namespace WebCore
Added: trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataConverter.mm (0 => 287157)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataConverter.mm (rev 0)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataConverter.mm 2021-12-16 22:02:54 UTC (rev 287157)
@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#import "config.h"
+#import "AudioSampleDataConverter.h"
+
+#import "AudioSampleBufferList.h"
+#import <AudioToolbox/AudioConverter.h>
+#import <pal/cf/AudioToolboxSoftLink.h>
+
+namespace WebCore {
+
+AudioSampleDataConverter::~AudioSampleDataConverter()
+{
+}
+
+OSStatus AudioSampleDataConverter::setFormats(const CAAudioStreamDescription& inputDescription, const CAAudioStreamDescription& outputDescription)
+{
+ constexpr double buffer100ms = 0.100;
+ constexpr double buffer60ms = 0.060;
+ constexpr double buffer50ms = 0.050;
+ constexpr double buffer40ms = 0.040;
+ constexpr double buffer20ms = 0.020;
+ m_highBufferSize = outputDescription.sampleRate() * buffer100ms;
+ m_regularHighBufferSize = outputDescription.sampleRate() * buffer60ms;
+ m_regularBufferSize = outputDescription.sampleRate() * buffer50ms;
+ m_regularLowBufferSize = outputDescription.sampleRate() * buffer40ms;
+ m_lowBufferSize = outputDescription.sampleRate() * buffer20ms;
+
+ m_selectedConverter = nullptr;
+
+ auto converterOutputDescription = outputDescription.streamDescription();
+ constexpr double slightlyHigherPitch = 1.05;
+ converterOutputDescription.mSampleRate = slightlyHigherPitch * outputDescription.streamDescription().mSampleRate;
+ if (auto error = m_lowConverter.initialize(inputDescription.streamDescription(), converterOutputDescription); error != noErr)
+ return error;
+
+ constexpr double slightlyLowerPitch = 0.95;
+ converterOutputDescription.mSampleRate = slightlyLowerPitch * outputDescription.streamDescription().mSampleRate;
+ if (auto error = m_highConverter.initialize(inputDescription.streamDescription(), converterOutputDescription); error != noErr)
+ return error;
+
+ if (inputDescription == outputDescription)
+ return noErr;
+
+ if (auto error = m_regularConverter.initialize(inputDescription.streamDescription(), outputDescription.streamDescription()); error != noErr)
+ return error;
+
+ m_selectedConverter = m_regularConverter;
+ return noErr;
+}
+
+bool AudioSampleDataConverter::updateBufferedAmount(size_t currentBufferedAmount)
+{
+ if (currentBufferedAmount) {
+ if (m_selectedConverter == m_regularConverter) {
+ if (currentBufferedAmount <= m_lowBufferSize)
+ m_selectedConverter = m_lowConverter;
+ else if (currentBufferedAmount >= m_highBufferSize)
+ m_selectedConverter = m_highConverter;
+ } else if (m_selectedConverter == m_highConverter) {
+ if (currentBufferedAmount < m_regularLowBufferSize)
+ m_selectedConverter = m_regularConverter;
+ } else if (currentBufferedAmount > m_regularHighBufferSize)
+ m_selectedConverter = m_regularConverter;
+ }
+ return !!m_selectedConverter;
+}
+
+OSStatus AudioSampleDataConverter::convert(const AudioBufferList& inputBuffer, AudioSampleBufferList& outputBuffer, size_t sampleCount)
+{
+ outputBuffer.reset();
+ return outputBuffer.copyFrom(inputBuffer, sampleCount, m_selectedConverter);
+}
+
+OSStatus AudioSampleDataConverter::Converter::initialize(const AudioStreamBasicDescription& inputDescription, const AudioStreamBasicDescription& outputDescription)
+{
+ if (m_audioConverter) {
+ PAL::AudioConverterDispose(m_audioConverter);
+ m_audioConverter = nullptr;
+ }
+
+ return PAL::AudioConverterNew(&inputDescription, &outputDescription, &m_audioConverter);
+}
+
+AudioSampleDataConverter::Converter::~Converter()
+{
+ if (m_audioConverter)
+ PAL::AudioConverterDispose(m_audioConverter);
+}
+
+} // namespace WebCore
Modified: trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.h (287156 => 287157)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.h 2021-12-16 22:00:44 UTC (rev 287156)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.h 2021-12-16 22:02:54 UTC (rev 287157)
@@ -25,6 +25,7 @@
#pragma once
+#include "AudioSampleDataConverter.h"
#include "CARingBuffer.h"
#include <CoreAudio/CoreAudioTypes.h>
#include <wtf/LoggerHelper.h>
@@ -102,13 +103,15 @@
uint64_t m_lastPushedSampleCount { 0 };
size_t m_waitToStartForPushCount { 2 };
- MediaTime m_expectedNextPushedSampleTime { MediaTime::invalidTime() };
- bool m_isFirstPull { true };
- MediaTime m_inputSampleOffset;
+ int64_t m_expectedNextPushedSampleTimeValue { 0 };
+ int64_t m_converterInputOffset { 0 };
+ std::optional<int64_t> m_inputSampleOffset;
int64_t m_outputSampleOffset { 0 };
+ uint64_t m_lastBufferedAmount { 0 };
- AudioConverterRef m_converter;
+ AudioSampleDataConverter m_converter;
+
RefPtr<AudioSampleBufferList> m_scratchBuffer;
UniqueRef<CARingBuffer> m_ringBuffer;
@@ -117,10 +120,8 @@
float m_volume { 1.0 };
bool m_muted { false };
bool m_shouldComputeOutputSampleOffset { true };
- uint64_t m_endFrameWhenNotEnoughData { 0 };
bool m_isInNeedOfMoreData { false };
-
#if !RELEASE_LOG_DISABLED
Ref<const Logger> m_logger;
const void* m_logIdentifier;
Modified: trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm (287156 => 287157)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm 2021-12-16 22:00:44 UTC (rev 287156)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm 2021-12-16 22:02:54 UTC (rev 287157)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2017-2021 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -51,7 +51,6 @@
AudioSampleDataSource::AudioSampleDataSource(size_t maximumSampleCount, LoggerHelper& loggerHelper, size_t waitToStartForPushCount)
: m_waitToStartForPushCount(waitToStartForPushCount)
- , m_inputSampleOffset(MediaTime::invalidTime())
, m_ringBuffer(makeUniqueRef<CARingBuffer>())
, m_maximumSampleCount(maximumSampleCount)
#if !RELEASE_LOG_DISABLED
@@ -66,31 +65,12 @@
AudioSampleDataSource::~AudioSampleDataSource()
{
- if (m_converter)
- PAL::AudioConverterDispose(m_converter);
}
OSStatus AudioSampleDataSource::setupConverter()
{
ASSERT(m_inputDescription && m_outputDescription);
-
- if (m_converter) {
- PAL::AudioConverterDispose(m_converter);
- m_converter = nullptr;
- }
-
- if (*m_inputDescription == *m_outputDescription)
- return 0;
-
- OSStatus err = PAL::AudioConverterNew(&m_inputDescription->streamDescription(), &m_outputDescription->streamDescription(), &m_converter);
- if (err) {
- RunLoop::main().dispatch([this, protectedThis = Ref { *this }, err] {
- ERROR_LOG("AudioConverterNew returned error ", err);
- });
- }
-
- return err;
-
+ return m_converter.setFormats(*m_inputDescription, *m_outputDescription);
}
OSStatus AudioSampleDataSource::setInputFormat(const CAAudioStreamDescription& format)
@@ -109,6 +89,9 @@
ASSERT(m_inputDescription);
ASSERT(format.sampleRate() >= 0);
+ if (m_outputDescription && *m_outputDescription == format)
+ return noErr;
+
m_outputDescription = CAAudioStreamDescription { format };
{
@@ -117,6 +100,7 @@
DisableMallocRestrictionsForCurrentThreadScope disableMallocRestrictions;
m_ringBuffer->allocate(format, static_cast<size_t>(m_maximumSampleCount));
m_scratchBuffer = AudioSampleBufferList::create(m_outputDescription->streamDescription(), m_maximumSampleCount);
+ m_converterInputOffset = 0;
}
return setupConverter();
@@ -140,35 +124,45 @@
void AudioSampleDataSource::pushSamplesInternal(const AudioBufferList& bufferList, const MediaTime& presentationTime, size_t sampleCount)
{
- MediaTime sampleTime = presentationTime;
+ int64_t ringBufferIndexToWrite = presentationTime.toTimeScale(m_outputDescription->sampleRate()).timeValue();
+ int64_t offset = 0;
const AudioBufferList* sampleBufferList;
- if (m_converter) {
+
+ if (m_converter.updateBufferedAmount(m_lastBufferedAmount)) {
m_scratchBuffer->reset();
- OSStatus err = m_scratchBuffer->copyFrom(bufferList, sampleCount, m_converter);
- if (err)
- return;
+ m_converter.convert(bufferList, *m_scratchBuffer, sampleCount);
+ auto expectedSampleCount = sampleCount * m_outputDescription->sampleRate() / m_inputDescription->sampleRate();
+ if (m_converter.isRegular() && expectedSampleCount > m_scratchBuffer->sampleCount()) {
+ // Sometimes converter is not writing enough data, for instance on first chunk conversion.
+ // Pretend this is the case to keep pusher and puller in sync.
+ offset = 0;
+ sampleCount = expectedSampleCount;
+ if (m_scratchBuffer->sampleCount() > sampleCount)
+ m_scratchBuffer->setSampleCount(sampleCount);
+ } else {
+ offset = m_scratchBuffer->sampleCount() - expectedSampleCount;
+ sampleCount = m_scratchBuffer->sampleCount();
+ }
sampleBufferList = m_scratchBuffer->bufferList().list();
- sampleCount = m_scratchBuffer->sampleCount();
- sampleTime = presentationTime.toTimeScale(m_outputDescription->sampleRate(), MediaTime::RoundingFlags::TowardZero);
} else
sampleBufferList = &bufferList;
- if (m_expectedNextPushedSampleTime.isValid() && abs(m_expectedNextPushedSampleTime - sampleTime).timeValue() == 1)
- sampleTime = m_expectedNextPushedSampleTime;
- m_expectedNextPushedSampleTime = sampleTime + MediaTime(sampleCount, sampleTime.timeScale());
+ if (!m_inputSampleOffset) {
+ m_inputSampleOffset = 0 - ringBufferIndexToWrite;
+ ringBufferIndexToWrite = 0;
+ } else
+ ringBufferIndexToWrite += *m_inputSampleOffset;
- if (m_inputSampleOffset == MediaTime::invalidTime())
- m_inputSampleOffset = MediaTime(1 - sampleTime.timeValue(), sampleTime.timeScale());
- sampleTime += m_inputSampleOffset;
+ if (m_converterInputOffset)
+ ringBufferIndexToWrite += m_converterInputOffset;
-#if !LOG_DISABLED
- uint64_t startFrame1 = 0;
- uint64_t endFrame1 = 0;
- m_ringBuffer->getCurrentFrameBounds(startFrame1, endFrame1);
-#endif
+ if (m_expectedNextPushedSampleTimeValue && abs((float)m_expectedNextPushedSampleTimeValue - (float)ringBufferIndexToWrite) <= 1)
+ ringBufferIndexToWrite = m_expectedNextPushedSampleTimeValue;
+ m_expectedNextPushedSampleTimeValue = ringBufferIndexToWrite + sampleCount;
+
if (m_isInNeedOfMoreData) {
m_isInNeedOfMoreData = false;
DisableMallocRestrictionsForCurrentThreadScope disableMallocRestrictions;
@@ -176,7 +170,10 @@
ALWAYS_LOG(logIdentifier, "needed more data, pushing ", sampleCount, " samples");
});
}
- m_ringBuffer->store(sampleBufferList, sampleCount, sampleTime.timeValue());
+
+ m_ringBuffer->store(sampleBufferList, sampleCount, ringBufferIndexToWrite);
+
+ m_converterInputOffset += offset;
m_lastPushedSampleCount = sampleCount;
}
@@ -194,21 +191,6 @@
pushSamplesInternal(*downcast<WebAudioBufferList>(audioData).list(), sampleTime, sampleCount);
}
-static inline int64_t computeOffsetDelay(double sampleRate, uint64_t lastPushedSampleCount)
-{
- const double twentyMS = .02;
- const double tenMS = .01;
- const double fiveMS = .005;
-
- if (lastPushedSampleCount > sampleRate * twentyMS)
- return sampleRate * twentyMS;
- if (lastPushedSampleCount > sampleRate * tenMS)
- return sampleRate * tenMS;
- if (lastPushedSampleCount > sampleRate * fiveMS)
- return sampleRate * fiveMS;
- return 0;
-}
-
bool AudioSampleDataSource::pullSamples(AudioBufferList& buffer, size_t sampleCount, uint64_t timeStamp, double /*hostTime*/, PullMode mode)
{
size_t byteCount = sampleCount * m_outputDescription->bytesPerFrame();
@@ -220,7 +202,7 @@
return false;
}
- if (m_muted || m_inputSampleOffset == MediaTime::invalidTime()) {
+ if (m_muted || !m_inputSampleOffset) {
if (mode != AudioSampleDataSource::Mix)
AudioSampleBufferList::zeroABL(buffer, byteCount);
return false;
@@ -230,33 +212,19 @@
uint64_t endFrame = 0;
m_ringBuffer->getCurrentFrameBounds(startFrame, endFrame);
+ ASSERT(m_waitToStartForPushCount);
+
+ uint64_t buffered = endFrame - startFrame;
if (m_shouldComputeOutputSampleOffset) {
- uint64_t buffered = endFrame - startFrame;
- if (m_isFirstPull) {
- auto minimumBuffer = m_waitToStartForPushCount * m_lastPushedSampleCount;
- if (buffered >= minimumBuffer) {
- m_outputSampleOffset = startFrame - timeStamp;
- m_shouldComputeOutputSampleOffset = false;
- m_endFrameWhenNotEnoughData = 0;
- } else {
- // We wait for one chunk of value before starting to play.
- if (mode != AudioSampleDataSource::Mix)
- AudioSampleBufferList::zeroABL(buffer, byteCount);
- return false;
- }
- } else {
- if (buffered < sampleCount * 2 || (m_endFrameWhenNotEnoughData && m_endFrameWhenNotEnoughData == endFrame)) {
- if (mode != AudioSampleDataSource::Mix)
- AudioSampleBufferList::zeroABL(buffer, byteCount);
- return false;
- }
-
- m_shouldComputeOutputSampleOffset = false;
- m_endFrameWhenNotEnoughData = 0;
-
- m_outputSampleOffset = (endFrame - sampleCount) - timeStamp;
- m_outputSampleOffset -= computeOffsetDelay(m_outputDescription->sampleRate(), m_lastPushedSampleCount);
+ auto minimumBuffer = std::max<size_t>(m_waitToStartForPushCount * m_lastPushedSampleCount, m_converter.regularBufferSize());
+ if (buffered < minimumBuffer) {
+ // We wait for one chunk of value before starting to play.
+ if (mode != AudioSampleDataSource::Mix)
+ AudioSampleBufferList::zeroABL(buffer, byteCount);
+ return false;
}
+ m_outputSampleOffset = endFrame - timeStamp - minimumBuffer;
+ m_shouldComputeOutputSampleOffset = false;
}
timeStamp += m_outputSampleOffset;
@@ -269,23 +237,13 @@
ERROR_LOG(logIdentifier, "need more data, sample ", timeStamp, " with offset ", outputSampleOffset, ", trying to get ", sampleCount, " samples, but not completely in range [", startFrame, " .. ", endFrame, "]");
});
}
- if (timeStamp < startFrame || timeStamp >= endFrame) {
- // We are out of the window, let's restart the offset computation.
- m_shouldComputeOutputSampleOffset = true;
-
- if (timeStamp >= endFrame)
- m_endFrameWhenNotEnoughData = endFrame;
- } else {
- // We are too close from endFrame, let's wait for more data to be pushed.
- m_outputSampleOffset -= sampleCount;
- }
+ m_shouldComputeOutputSampleOffset = true;
if (mode != AudioSampleDataSource::Mix)
AudioSampleBufferList::zeroABL(buffer, byteCount);
return false;
}
- m_isFirstPull = false;
-
+ m_lastBufferedAmount = endFrame - timeStamp - sampleCount;
return pullSamplesInternal(buffer, sampleCount, timeStamp, mode);
}
@@ -320,12 +278,12 @@
if (buffer.mNumberBuffers != m_ringBuffer->channelCount())
return false;
- if (m_muted)
+ if (m_muted || !m_inputSampleOffset)
return false;
if (m_shouldComputeOutputSampleOffset) {
m_shouldComputeOutputSampleOffset = false;
- m_outputSampleOffset = m_inputSampleOffset.timeValue() * m_outputDescription->sampleRate() / m_inputSampleOffset.timeScale();
+ m_outputSampleOffset = *m_inputSampleOffset;
}
timeStamp += m_outputSampleOffset;
@@ -354,6 +312,10 @@
startFrame = timeStamp;
+ ASSERT(endFrame >= startFrame);
+ if (endFrame < startFrame)
+ return false;
+
if (m_muted) {
AudioSampleBufferList::zeroABL(buffer, sampleCountPerChunk * m_outputDescription->bytesPerFrame());
while (endFrame - startFrame >= sampleCountPerChunk) {
Modified: trunk/Source/WebCore/platform/mediastream/mac/MockAudioSharedUnit.h (287156 => 287157)
--- trunk/Source/WebCore/platform/mediastream/mac/MockAudioSharedUnit.h 2021-12-16 22:00:44 UTC (rev 287156)
+++ trunk/Source/WebCore/platform/mediastream/mac/MockAudioSharedUnit.h 2021-12-16 22:02:54 UTC (rev 287157)
@@ -62,15 +62,16 @@
void delaySamples(Seconds) final;
+ void start();
CapabilityValueOrRange sampleRateCapacities() const final { return CapabilityValueOrRange(44100, 48000); }
void tick();
- void render(Seconds);
+ void render(MonotonicTime);
void emitSampleBuffers(uint32_t frameCount);
void reconfigure();
- static Seconds renderInterval() { return 60_ms; }
+ static Seconds renderInterval() { return 20_ms; }
std::unique_ptr<WebAudioBufferList> m_audioBufferList;
@@ -83,6 +84,7 @@
Vector<float> m_bipBopBuffer;
bool m_hasAudioUnit { false };
+ bool m_isProducingData { false };
RunLoop::Timer<MockAudioSharedUnit> m_timer;
MonotonicTime m_lastRenderTime { MonotonicTime::nan() };
Modified: trunk/Source/WebCore/platform/mediastream/mac/MockAudioSharedUnit.mm (287156 => 287157)
--- trunk/Source/WebCore/platform/mediastream/mac/MockAudioSharedUnit.mm 2021-12-16 22:00:44 UTC (rev 287156)
+++ trunk/Source/WebCore/platform/mediastream/mac/MockAudioSharedUnit.mm 2021-12-16 22:02:54 UTC (rev 287157)
@@ -101,8 +101,8 @@
}
MockAudioSharedUnit::MockAudioSharedUnit()
- : m_timer(RunLoop::current(), this, &MockAudioSharedUnit::tick)
- , m_workQueue(WorkQueue::create("MockAudioSharedUnit Capture Queue"))
+ : m_timer(RunLoop::current(), this, &MockAudioSharedUnit::start)
+ , m_workQueue(WorkQueue::create("MockAudioSharedUnit Capture Queue", WorkQueue::QOS::UserInteractive))
{
}
@@ -127,13 +127,11 @@
if (!hasAudioUnit())
return 0;
- m_timer.stop();
m_lastRenderTime = MonotonicTime::nan();
m_workQueue->dispatch([this] {
reconfigure();
callOnMainThread([this] {
- m_lastRenderTime = MonotonicTime::now();
- m_timer.startRepeating(renderInterval());
+ startInternal();
});
});
return 0;
@@ -142,57 +140,45 @@
void MockAudioSharedUnit::cleanupAudioUnit()
{
m_hasAudioUnit = false;
- m_timer.stop();
+ m_isProducingData = false;
m_lastRenderTime = MonotonicTime::nan();
}
OSStatus MockAudioSharedUnit::startInternal()
{
+ start();
+ return 0;
+}
+
+void MockAudioSharedUnit::start()
+{
if (!m_hasAudioUnit)
m_hasAudioUnit = true;
m_lastRenderTime = MonotonicTime::now();
- m_timer.startRepeating(renderInterval());
- return 0;
+ m_isProducingData = true;
+ m_workQueue->dispatch([this, renderTime = m_lastRenderTime] {
+ render(renderTime);
+ });
}
void MockAudioSharedUnit::stopInternal()
{
+ m_isProducingData = false;
if (!m_hasAudioUnit)
return;
- m_timer.stop();
m_lastRenderTime = MonotonicTime::nan();
}
bool MockAudioSharedUnit::isProducingData() const
{
- return m_timer.isActive();
+ return m_isProducingData;
}
-void MockAudioSharedUnit::tick()
-{
- if (std::isnan(m_lastRenderTime))
- m_lastRenderTime = MonotonicTime::now();
-
- MonotonicTime now = MonotonicTime::now();
-
- if (m_delayUntil) {
- if (m_delayUntil < now)
- return;
- m_delayUntil = MonotonicTime();
- }
-
- Seconds delta = now - m_lastRenderTime;
- m_lastRenderTime = now;
-
- m_workQueue->dispatch([this, delta] {
- render(delta);
- });
-}
-
void MockAudioSharedUnit::delaySamples(Seconds delta)
{
- m_delayUntil = MonotonicTime::now() + delta;
+ stopInternal();
+ m_timer.startOneShot(delta);
}
void MockAudioSharedUnit::reconfigure()
@@ -244,9 +230,24 @@
audioSamplesAvailable(PAL::toMediaTime(startTime), *m_audioBufferList, CAAudioStreamDescription(m_streamFormat), frameCount);
}
-void MockAudioSharedUnit::render(Seconds delta)
+void MockAudioSharedUnit::render(MonotonicTime renderTime)
{
ASSERT(!isMainThread());
+ if (!isProducingData())
+ return;
+
+ auto delta = renderInterval();
+ auto currentTime = MonotonicTime::now();
+ auto nextRenderTime = renderTime + delta;
+ Seconds nextRenderDelay = nextRenderTime.secondsSinceEpoch() - currentTime.secondsSinceEpoch();
+ if (nextRenderDelay.seconds() < 0) {
+ nextRenderTime = currentTime;
+ nextRenderDelay = 0_s;
+ }
+ m_workQueue->dispatchAfter(nextRenderDelay, [this, nextRenderTime] {
+ render(nextRenderTime);
+ });
+
if (!m_audioBufferList || !m_bipBopBuffer.size())
reconfigure();
