Diff
Modified: trunk/Source/WebCore/ChangeLog (285026 => 285027)
--- trunk/Source/WebCore/ChangeLog 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/ChangeLog 2021-10-29 13:36:03 UTC (rev 285027)
@@ -1,5 +1,48 @@
2021-10-29 Youenn Fablet <[email protected]>
+ Ensure synchronized rendering of incoming audio tracks
+ https://bugs.webkit.org/show_bug.cgi?id=232375
+
+ Reviewed by Eric Carlson.
+
+ WebRTC incoming audio tracks are guaranteed to be synchronized based on the audio module clock.
+ Previously, we were mixing all audio tracks together, but we were not handling synchronization of tracks that have the same clock.
+ This can cause some lower quality audio rendering.
+ To prevent this, all WebRTC incoming audio tracks are now mixed together in a single track that is then mixed with all other non synchronized tracks.
+ To do so we introduce IncomingAudioMediaStreamTrackRendererUnit which will receive all incoming audio tracks.
+ It then sends the mix to the actual AudioMediaStreamTrackRendererUnit.
+ To ensure tight synchronization, we introduce AudioSampleDataSource::pullAvailableSampleChunk which reads data using the provided timestamp relative to the input offset.
+ We do a refactoring to make it easy to either use IncomingAudioMediaStreamTrackRendererUnit or AudioMediaStreamTrackRendererUnit by having a base class called BaseAudioMediaStreamTrackRendererUnit.
+
+ Manually tested.
+
+ * Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.cpp:
+ * Modules/mediastream/libwebrtc/LibWebRTCRtpReceiverBackend.cpp:
+ * Modules/mediastream/libwebrtc/LibWebRTCRtpReceiverBackend.h:
+ * SourcesCocoa.txt:
+ * WebCore.xcodeproj/project.pbxproj:
+ * loader/EmptyClients.cpp:
+ * platform/audio/cocoa/AudioSampleDataSource.h:
+ * platform/audio/cocoa/AudioSampleDataSource.mm:
+ * platform/mediastream/AudioMediaStreamTrackRenderer.cpp:
+ * platform/mediastream/AudioMediaStreamTrackRenderer.h:
+ * platform/mediastream/AudioTrackPrivateMediaStream.cpp:
+ * platform/mediastream/RealtimeIncomingAudioSource.cpp:
+ * platform/mediastream/RealtimeIncomingAudioSource.h:
+ * platform/mediastream/cocoa/AudioMediaStreamTrackRendererCocoa.cpp:
+ * platform/mediastream/cocoa/AudioMediaStreamTrackRendererCocoa.h:
+ * platform/mediastream/cocoa/AudioMediaStreamTrackRendererUnit.h:
+ * platform/mediastream/cocoa/BaseAudioMediaStreamTrackRendererUnit.h: Added.
+ * platform/mediastream/cocoa/IncomingAudioMediaStreamTrackRendererUnit.cpp: Added.
+ * platform/mediastream/cocoa/IncomingAudioMediaStreamTrackRendererUnit.h: Added.
+ * platform/mediastream/libwebrtc/LibWebRTCAudioModule.cpp:
+ * platform/mediastream/libwebrtc/LibWebRTCAudioModule.h:
+ * platform/mediastream/libwebrtc/LibWebRTCProvider.cpp:
+ * platform/mediastream/libwebrtc/LibWebRTCProvider.h:
+ * platform/mediastream/mac/RealtimeIncomingAudioSourceCocoa.cpp:
+
+2021-10-29 Youenn Fablet <[email protected]>
+
Fix potential race in AudioMediaStreamTrackRendererCocoa::pushSamples
https://bugs.webkit.org/show_bug.cgi?id=232430
Modified: trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.cpp (285026 => 285027)
--- trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.cpp 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.cpp 2021-10-29 13:36:03 UTC (rev 285027)
@@ -29,8 +29,10 @@
#include "Document.h"
#include "IceCandidate.h"
+#include "LibWebRTCAudioModule.h"
#include "LibWebRTCDataChannelHandler.h"
#include "LibWebRTCMediaEndpoint.h"
+#include "LibWebRTCProvider.h"
#include "LibWebRTCRtpReceiverBackend.h"
#include "LibWebRTCRtpSenderBackend.h"
#include "LibWebRTCRtpTransceiverBackend.h"
@@ -293,7 +295,8 @@
{
auto& document = downcast<Document>(*m_peerConnection.scriptExecutionContext());
- auto source = backend->createSource();
+ auto source = backend->createSource(document);
+
// Remote source is initially muted and will be unmuted when receiving the first packet.
source->setMuted(true);
auto trackID = source->persistentID();
Modified: trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpReceiverBackend.cpp (285026 => 285027)
--- trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpReceiverBackend.cpp 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpReceiverBackend.cpp 2021-10-29 13:36:03 UTC (rev 285027)
@@ -25,9 +25,13 @@
#include "config.h"
#include "LibWebRTCRtpReceiverBackend.h"
+#include "Document.h"
+#include "LibWebRTCAudioModule.h"
#include "LibWebRTCDtlsTransportBackend.h"
+#include "LibWebRTCProvider.h"
#include "LibWebRTCRtpReceiverTransformBackend.h"
#include "LibWebRTCUtils.h"
+#include "Page.h"
#include "RTCRtpTransformBackend.h"
#include "RealtimeIncomingAudioSource.h"
#include "RealtimeIncomingVideoSource.h"
@@ -99,7 +103,7 @@
return sources;
}
-Ref<RealtimeMediaSource> LibWebRTCRtpReceiverBackend::createSource()
+Ref<RealtimeMediaSource> LibWebRTCRtpReceiverBackend::createSource(Document& document)
{
auto rtcTrack = m_rtcReceiver->track();
switch (m_rtcReceiver->media_type()) {
@@ -108,7 +112,10 @@
break;
case cricket::MEDIA_TYPE_AUDIO: {
rtc::scoped_refptr<webrtc::AudioTrackInterface> audioTrack = static_cast<webrtc::AudioTrackInterface*>(rtcTrack.get());
- return RealtimeIncomingAudioSource::create(WTFMove(audioTrack), fromStdString(rtcTrack->id()));
+ auto source = RealtimeIncomingAudioSource::create(WTFMove(audioTrack), fromStdString(rtcTrack->id()));
+ if (document.page())
+ source->setAudioModule(document.page()->libWebRTCProvider().audioModule());
+ return source;
}
case cricket::MEDIA_TYPE_VIDEO: {
rtc::scoped_refptr<webrtc::VideoTrackInterface> videoTrack = static_cast<webrtc::VideoTrackInterface*>(rtcTrack.get());
Modified: trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpReceiverBackend.h (285026 => 285027)
--- trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpReceiverBackend.h 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpReceiverBackend.h 2021-10-29 13:36:03 UTC (rev 285027)
@@ -35,7 +35,7 @@
}
namespace WebCore {
-
+class Document;
class RealtimeMediaSource;
class LibWebRTCRtpReceiverBackend final : public RTCRtpReceiverBackend {
@@ -46,7 +46,7 @@
webrtc::RtpReceiverInterface* rtcReceiver() { return m_rtcReceiver.get(); }
- Ref<RealtimeMediaSource> createSource();
+ Ref<RealtimeMediaSource> createSource(Document&);
private:
RTCRtpParameters getParameters() final;
Modified: trunk/Source/WebCore/SourcesCocoa.txt (285026 => 285027)
--- trunk/Source/WebCore/SourcesCocoa.txt 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/SourcesCocoa.txt 2021-10-29 13:36:03 UTC (rev 285027)
@@ -526,6 +526,7 @@
platform/mediastream/cocoa/AudioMediaStreamTrackRendererInternalUnit.cpp
platform/mediastream/cocoa/AudioMediaStreamTrackRendererUnit.cpp
platform/mediastream/cocoa/DisplayCaptureSourceCocoa.cpp
+platform/mediastream/cocoa/IncomingAudioMediaStreamTrackRendererUnit.cpp
platform/mediastream/ios/AVAudioSessionCaptureDevice.mm
platform/mediastream/ios/AVAudioSessionCaptureDeviceManager.mm @no-unify
platform/mediastream/ios/CoreAudioCaptureSourceIOS.mm
Modified: trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj (285026 => 285027)
--- trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2021-10-29 13:36:03 UTC (rev 285027)
@@ -1180,6 +1180,7 @@
4186BD3F213EE3430001826F /* LibWebRTCRtpSenderBackend.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 41D1A04B213EDDFE0063FB6B /* LibWebRTCRtpSenderBackend.cpp */; };
4186BD40213EE3450001826F /* LibWebRTCRtpReceiverBackend.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 41D1A04A213EDDFE0063FB6B /* LibWebRTCRtpReceiverBackend.cpp */; };
4186BD4E2140B9E80001826F /* LibWebRTCRtpTransceiverBackend.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 4186BD4D2140B9E80001826F /* LibWebRTCRtpTransceiverBackend.cpp */; };
+ 41878FAD27294721002E1EDD /* BaseAudioMediaStreamTrackRendererUnit.h in Headers */ = {isa = PBXBuildFile; fileRef = 41878FA827282055002E1EDD /* BaseAudioMediaStreamTrackRendererUnit.h */; settings = {ATTRIBUTES = (Private, ); }; };
41885B9311B6FDA6003383BB /* FormSubmission.h in Headers */ = {isa = PBXBuildFile; fileRef = 41885B9111B6FDA6003383BB /* FormSubmission.h */; settings = {ATTRIBUTES = (Private, ); }; };
4188D55C26C54D6F004858C8 /* RTCDtlsTransportBackend.h in Headers */ = {isa = PBXBuildFile; fileRef = 4188D55A26C54D6F004858C8 /* RTCDtlsTransportBackend.h */; };
4188D56326C551B1004858C8 /* RTCDtlsTransportState.h in Headers */ = {isa = PBXBuildFile; fileRef = 4188D56226C551B1004858C8 /* RTCDtlsTransportState.h */; };
@@ -8158,6 +8159,9 @@
4186BD46214072B60001826F /* RTCRtpTransceiverBackend.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RTCRtpTransceiverBackend.h; sourceTree = "<group>"; };
4186BD4B2140A8050001826F /* LibWebRTCRtpTransceiverBackend.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = LibWebRTCRtpTransceiverBackend.h; path = libwebrtc/LibWebRTCRtpTransceiverBackend.h; sourceTree = "<group>"; };
4186BD4D2140B9E80001826F /* LibWebRTCRtpTransceiverBackend.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LibWebRTCRtpTransceiverBackend.cpp; path = libwebrtc/LibWebRTCRtpTransceiverBackend.cpp; sourceTree = "<group>"; };
+ 41878FA827282055002E1EDD /* BaseAudioMediaStreamTrackRendererUnit.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = BaseAudioMediaStreamTrackRendererUnit.h; sourceTree = "<group>"; };
+ 41878FAA27282057002E1EDD /* IncomingAudioMediaStreamTrackRendererUnit.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = IncomingAudioMediaStreamTrackRendererUnit.h; sourceTree = "<group>"; };
+ 41878FAB272934E6002E1EDD /* IncomingAudioMediaStreamTrackRendererUnit.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = IncomingAudioMediaStreamTrackRendererUnit.cpp; sourceTree = "<group>"; };
418807DF24E4558300DDAF94 /* AbortAlgorithm.idl */ = {isa = PBXFileReference; lastKnownFileType = text; path = AbortAlgorithm.idl; sourceTree = "<group>"; };
418807E124E458C300DDAF94 /* AbortAlgorithm.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AbortAlgorithm.h; sourceTree = "<group>"; };
41885B9111B6FDA6003383BB /* FormSubmission.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FormSubmission.h; sourceTree = "<group>"; };
@@ -20172,8 +20176,11 @@
41C3B89F2649B1B7004ED4DE /* AudioMediaStreamTrackRendererInternalUnit.h */,
41C3B8A02649B1B7004ED4DE /* AudioMediaStreamTrackRendererUnit.cpp */,
41C3B89B2649B1B6004ED4DE /* AudioMediaStreamTrackRendererUnit.h */,
+ 41878FA827282055002E1EDD /* BaseAudioMediaStreamTrackRendererUnit.h */,
07BB1E6E27176CCA001DF289 /* DisplayCaptureSourceCocoa.cpp */,
07BB1E6F27176CCA001DF289 /* DisplayCaptureSourceCocoa.h */,
+ 41878FAB272934E6002E1EDD /* IncomingAudioMediaStreamTrackRendererUnit.cpp */,
+ 41878FAA27282057002E1EDD /* IncomingAudioMediaStreamTrackRendererUnit.h */,
);
path = cocoa;
sourceTree = "<group>";
@@ -31928,6 +31935,7 @@
BC124EE80C2641CD009E2349 /* BarProp.h in Headers */,
460BB6161D0A1BF000221812 /* Base64Utilities.h in Headers */,
83198FBF24A160DD00420B05 /* BaseAudioContext.h in Headers */,
+ 41878FAD27294721002E1EDD /* BaseAudioMediaStreamTrackRendererUnit.h in Headers */,
412DEF1F23A918A300D840F6 /* BaseAudioSharedUnit.h in Headers */,
379E61CA126CA5C400B63E8D /* BaseButtonInputType.h in Headers */,
379E61CC126CA5C400B63E8D /* BaseCheckableInputType.h in Headers */,
Modified: trunk/Source/WebCore/loader/EmptyClients.cpp (285026 => 285027)
--- trunk/Source/WebCore/loader/EmptyClients.cpp 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/loader/EmptyClients.cpp 2021-10-29 13:36:03 UTC (rev 285027)
@@ -57,6 +57,7 @@
#include "HistoryItem.h"
#include "IDBConnectionToServer.h"
#include "InspectorClient.h"
+#include "LibWebRTCAudioModule.h"
#include "LibWebRTCProvider.h"
#include "MediaRecorderPrivate.h"
#include "MediaRecorderProvider.h"
Modified: trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.h (285026 => 285027)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.h 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.h 2021-10-29 13:36:03 UTC (rev 285027)
@@ -61,6 +61,7 @@
bool pullSamples(AudioBufferList&, size_t, uint64_t, double, PullMode);
bool pullAvailableSamplesAsChunks(AudioBufferList&, size_t frameCount, uint64_t timeStamp, Function<void()>&&);
+ bool pullAvailableSampleChunk(AudioBufferList&, size_t frameCount, uint64_t timeStamp, PullMode);
void setVolume(float volume) { m_volume = volume; }
float volume() const { return m_volume; }
@@ -69,6 +70,7 @@
bool muted() const { return m_muted; }
const CAAudioStreamDescription* inputDescription() const { return m_inputDescription ? &m_inputDescription.value() : nullptr; }
+ const CAAudioStreamDescription* outputDescription() const { return m_outputDescription ? &m_outputDescription.value() : nullptr; }
void recomputeSampleOffset() { m_shouldComputeOutputSampleOffset = true; }
@@ -86,6 +88,7 @@
OSStatus setupConverter();
void pushSamplesInternal(const AudioBufferList&, const MediaTime&, size_t frameCount);
+ bool pullSamplesInternal(AudioBufferList&, size_t sampleCount, uint64_t timeStamp, PullMode);
std::optional<CAAudioStreamDescription> m_inputDescription;
std::optional<CAAudioStreamDescription> m_outputDescription;
Modified: trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm (285026 => 285027)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm 2021-10-29 13:36:03 UTC (rev 285027)
@@ -286,6 +286,11 @@
m_isFirstPull = false;
+ return pullSamplesInternal(buffer, sampleCount, timeStamp, mode);
+}
+
+bool AudioSampleDataSource::pullSamplesInternal(AudioBufferList& buffer, size_t sampleCount, uint64_t timeStamp, PullMode mode)
+{
if (mode == Copy) {
m_ringBuffer->fetch(&buffer, sampleCount, timeStamp, CARingBuffer::Copy);
if (m_volume < EquivalentToMaxVolume)
@@ -309,6 +314,25 @@
return true;
}
+bool AudioSampleDataSource::pullAvailableSampleChunk(AudioBufferList& buffer, size_t sampleCount, uint64_t timeStamp, PullMode mode)
+{
+ ASSERT(buffer.mNumberBuffers == m_ringBuffer->channelCount());
+ if (buffer.mNumberBuffers != m_ringBuffer->channelCount())
+ return false;
+
+ if (m_muted)
+ return false;
+
+ if (m_shouldComputeOutputSampleOffset) {
+ m_shouldComputeOutputSampleOffset = false;
+ m_outputSampleOffset = m_inputSampleOffset.timeValue() * m_outputDescription->sampleRate() / m_inputSampleOffset.timeScale();
+ }
+
+ timeStamp += m_outputSampleOffset;
+
+ return pullSamplesInternal(buffer, sampleCount, timeStamp, mode);
+}
+
bool AudioSampleDataSource::pullAvailableSamplesAsChunks(AudioBufferList& buffer, size_t sampleCountPerChunk, uint64_t timeStamp, Function<void()>&& consumeFilledBuffer)
{
ASSERT(buffer.mNumberBuffers == m_ringBuffer->channelCount());
Modified: trunk/Source/WebCore/platform/mediastream/AudioMediaStreamTrackRenderer.cpp (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/AudioMediaStreamTrackRenderer.cpp 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/AudioMediaStreamTrackRenderer.cpp 2021-10-29 13:36:03 UTC (rev 285027)
@@ -40,31 +40,28 @@
namespace WebCore {
-AudioMediaStreamTrackRenderer::RendererCreator AudioMediaStreamTrackRenderer::m_rendererCreator = nullptr;
-void AudioMediaStreamTrackRenderer::setCreator(RendererCreator creator)
+std::unique_ptr<AudioMediaStreamTrackRenderer> AudioMediaStreamTrackRenderer::create(Init&& init)
{
- m_rendererCreator = creator;
-}
-
-std::unique_ptr<AudioMediaStreamTrackRenderer> AudioMediaStreamTrackRenderer::create()
-{
- if (m_rendererCreator)
- return m_rendererCreator();
-
#if PLATFORM(COCOA)
- return makeUnique<AudioMediaStreamTrackRendererCocoa>();
+ return makeUnique<AudioMediaStreamTrackRendererCocoa>(WTFMove(init));
#else
return nullptr;
#endif
}
+AudioMediaStreamTrackRenderer::AudioMediaStreamTrackRenderer(Init&& init)
+ : m_crashCallback(WTFMove(init.crashCallback))
+#if USE(LIBWEBRTC)
+ , m_audioModule(WTFMove(init.audioModule))
+#endif
#if !RELEASE_LOG_DISABLED
-void AudioMediaStreamTrackRenderer::setLogger(const Logger& logger, const void* identifier)
+ , m_logger(init.logger)
+ , m_logIdentifier(init.logIdentifier)
+#endif
{
- m_logger = &logger;
- m_logIdentifier = identifier;
}
+#if !RELEASE_LOG_DISABLED
WTFLogChannel& AudioMediaStreamTrackRenderer::logChannel() const
{
return LogMedia;
Modified: trunk/Source/WebCore/platform/mediastream/AudioMediaStreamTrackRenderer.h (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/AudioMediaStreamTrackRenderer.h 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/AudioMediaStreamTrackRenderer.h 2021-10-29 13:36:03 UTC (rev 285027)
@@ -37,12 +37,23 @@
namespace WebCore {
class AudioStreamDescription;
+class LibWebRTCAudioModule;
class PlatformAudioData;
class WEBCORE_EXPORT AudioMediaStreamTrackRenderer : public LoggerHelper {
WTF_MAKE_FAST_ALLOCATED;
public:
- static std::unique_ptr<AudioMediaStreamTrackRenderer> create();
+ struct Init {
+ Function<void()>&& crashCallback;
+#if USE(LIBWEBRTC)
+ RefPtr<LibWebRTCAudioModule> audioModule;
+#endif
+#if !RELEASE_LOG_DISABLED
+ const Logger& logger;
+ const void* logIdentifier;
+#endif
+ };
+ static std::unique_ptr<AudioMediaStreamTrackRenderer> create(Init&&);
virtual ~AudioMediaStreamTrackRenderer() = default;
virtual void start(CompletionHandler<void()>&&) = 0;
@@ -56,16 +67,9 @@
virtual void setAudioOutputDevice(const String&);
-#if !RELEASE_LOG_DISABLED
- void setLogger(const Logger&, const void*);
-#endif
+protected:
+ explicit AudioMediaStreamTrackRenderer(Init&&);
- using RendererCreator = std::unique_ptr<AudioMediaStreamTrackRenderer> (*)();
- static void setCreator(RendererCreator);
-
- void setCrashCallback(Function<void()>&& callback) { m_crashCallback = WTFMove(callback); }
-
-protected:
#if !RELEASE_LOG_DISABLED
const Logger& logger() const final;
const void* logIdentifier() const final;
@@ -74,17 +78,23 @@
WTFLogChannel& logChannel() const final;
#endif
+#if USE(LIBWEBRTC)
+ LibWebRTCAudioModule* audioModule();
+#endif
+
void crashed();
private:
- static RendererCreator m_rendererCreator;
-
// Main thread writable members
float m_volume { 1 };
Function<void()> m_crashCallback;
+#if USE(LIBWEBRTC)
+ RefPtr<LibWebRTCAudioModule> m_audioModule;
+#endif
+
#if !RELEASE_LOG_DISABLED
- RefPtr<const Logger> m_logger;
+ Ref<const Logger> m_logger;
const void* m_logIdentifier;
#endif
};
@@ -105,10 +115,17 @@
m_crashCallback();
}
+#if USE(LIBWEBRTC)
+inline LibWebRTCAudioModule* AudioMediaStreamTrackRenderer::audioModule()
+{
+ return m_audioModule.get();
+}
+#endif
+
#if !RELEASE_LOG_DISABLED
inline const Logger& AudioMediaStreamTrackRenderer::logger() const
{
- return *m_logger;
+ return m_logger.get();
}
Modified: trunk/Source/WebCore/platform/mediastream/AudioTrackPrivateMediaStream.cpp (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/AudioTrackPrivateMediaStream.cpp 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/AudioTrackPrivateMediaStream.cpp 2021-10-29 13:36:03 UTC (rev 285027)
@@ -29,7 +29,9 @@
#if ENABLE(MEDIA_STREAM)
#include "AudioMediaStreamTrackRenderer.h"
+#include "LibWebRTCAudioModule.h"
#include "Logging.h"
+#include "RealtimeIncomingAudioSource.h"
namespace WebCore {
@@ -48,20 +50,31 @@
clear();
}
+#if USE(LIBWEBRTC)
+static RefPtr<LibWebRTCAudioModule> audioModuleFromSource(RealtimeMediaSource& source)
+{
+ if (!is<RealtimeIncomingAudioSource>(source))
+ return nullptr;
+ return downcast<RealtimeIncomingAudioSource>(source).audioModule();
+}
+#endif
+
std::unique_ptr<AudioMediaStreamTrackRenderer> AudioTrackPrivateMediaStream::createRenderer(AudioTrackPrivateMediaStream& stream)
{
- auto renderer = AudioMediaStreamTrackRenderer::create();
- if (!renderer)
- return nullptr;
-#if !RELEASE_LOG_DISABLED
auto& track = stream.m_streamTrack.get();
- renderer->setLogger(track.logger(), track.logIdentifier());
+ return AudioMediaStreamTrackRenderer::create(AudioMediaStreamTrackRenderer::Init {
+ [stream = WeakPtr { stream }] {
+ if (stream)
+ stream->createNewRenderer();
+ }
+#if USE(LIBWEBRTC)
+ , audioModuleFromSource(stream.m_audioSource.get())
#endif
- renderer->setCrashCallback([stream = WeakPtr { stream }] {
- if (stream)
- stream->createNewRenderer();
+#if !RELEASE_LOG_DISABLED
+ , track.logger()
+ , track.logIdentifier()
+#endif
});
- return renderer;
}
void AudioTrackPrivateMediaStream::clear()
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeIncomingAudioSource.cpp (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/RealtimeIncomingAudioSource.cpp 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeIncomingAudioSource.cpp 2021-10-29 13:36:03 UTC (rev 285027)
@@ -34,6 +34,7 @@
#if USE(LIBWEBRTC)
#include "LibWebRTCAudioFormat.h"
+#include "LibWebRTCAudioModule.h"
#include "Logging.h"
namespace WebCore {
@@ -80,6 +81,12 @@
return m_currentSettings;
}
+void RealtimeIncomingAudioSource::setAudioModule(RefPtr<LibWebRTCAudioModule>&& audioModule)
+{
+ ASSERT(!m_audioModule);
+ m_audioModule = WTFMove(audioModule);
}
+}
+
#endif // USE(LIBWEBRTC)
Modified: trunk/Source/WebCore/platform/mediastream/RealtimeIncomingAudioSource.h (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/RealtimeIncomingAudioSource.h 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/RealtimeIncomingAudioSource.h 2021-10-29 13:36:03 UTC (rev 285027)
@@ -45,6 +45,8 @@
namespace WebCore {
+class LibWebRTCAudioModule;
+
class RealtimeIncomingAudioSource
: public RealtimeMediaSource
, private webrtc::AudioTrackSinkInterface
@@ -53,6 +55,9 @@
public:
static Ref<RealtimeIncomingAudioSource> create(rtc::scoped_refptr<webrtc::AudioTrackInterface>&&, String&&);
+ void setAudioModule(RefPtr<LibWebRTCAudioModule>&&);
+ LibWebRTCAudioModule* audioModule() { return m_audioModule.get(); }
+
protected:
RealtimeIncomingAudioSource(rtc::scoped_refptr<webrtc::AudioTrackInterface>&&, String&&);
~RealtimeIncomingAudioSource();
@@ -79,6 +84,7 @@
RealtimeMediaSourceSettings m_currentSettings;
rtc::scoped_refptr<webrtc::AudioTrackInterface> m_audioTrack;
+ RefPtr<LibWebRTCAudioModule> m_audioModule;
#if !RELEASE_LOG_DISABLED
mutable RefPtr<const Logger> m_logger;
Modified: trunk/Source/WebCore/platform/mediastream/cocoa/AudioMediaStreamTrackRendererCocoa.cpp (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/cocoa/AudioMediaStreamTrackRendererCocoa.cpp 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/cocoa/AudioMediaStreamTrackRendererCocoa.cpp 2021-10-29 13:36:03 UTC (rev 285027)
@@ -36,8 +36,9 @@
namespace WebCore {
-AudioMediaStreamTrackRendererCocoa::AudioMediaStreamTrackRendererCocoa()
- : m_resetObserver([this] { reset(); })
+AudioMediaStreamTrackRendererCocoa::AudioMediaStreamTrackRendererCocoa(Init&& init)
+ : AudioMediaStreamTrackRenderer(WTFMove(init))
+ , m_resetObserver([this] { reset(); })
{
}
@@ -59,12 +60,19 @@
});
}
+BaseAudioMediaStreamTrackRendererUnit& AudioMediaStreamTrackRendererCocoa::rendererUnit()
+{
+ if (auto* audioModule = this->audioModule())
+ return audioModule->incomingAudioMediaStreamTrackRendererUnit();
+ return AudioMediaStreamTrackRendererUnit::singleton();
+}
+
void AudioMediaStreamTrackRendererCocoa::stop()
{
ASSERT(isMainThread());
if (m_registeredDataSource)
- AudioMediaStreamTrackRendererUnit::singleton().removeSource(*m_registeredDataSource);
+ rendererUnit().removeSource(*m_registeredDataSource);
}
void AudioMediaStreamTrackRendererCocoa::clear()
@@ -95,7 +103,7 @@
void AudioMediaStreamTrackRendererCocoa::setAudioOutputDevice(const String& deviceId)
{
// FIXME: We should create a unit for ourselves here or use the default unit if deviceId is matching.
- AudioMediaStreamTrackRendererUnit::singleton().setAudioOutputDevice(deviceId);
+ rendererUnit().setAudioOutputDevice(deviceId);
m_shouldRecreateDataSource = true;
}
@@ -104,7 +112,7 @@
ASSERT(isMainThread());
if (m_registeredDataSource)
- AudioMediaStreamTrackRendererUnit::singleton().removeSource(*m_registeredDataSource);
+ rendererUnit().removeSource(*m_registeredDataSource);
if (!m_outputDescription)
return;
@@ -115,8 +123,8 @@
m_registeredDataSource->setLogger(logger(), logIdentifier());
m_registeredDataSource->setVolume(volume());
- AudioMediaStreamTrackRendererUnit::singleton().addResetObserver(m_resetObserver);
- AudioMediaStreamTrackRendererUnit::singleton().addSource(*m_registeredDataSource);
+ rendererUnit().addResetObserver(m_resetObserver);
+ rendererUnit().addSource(*m_registeredDataSource);
}
static unsigned pollSamplesCount()
Modified: trunk/Source/WebCore/platform/mediastream/cocoa/AudioMediaStreamTrackRendererCocoa.h (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/cocoa/AudioMediaStreamTrackRendererCocoa.h 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/cocoa/AudioMediaStreamTrackRendererCocoa.h 2021-10-29 13:36:03 UTC (rev 285027)
@@ -39,12 +39,13 @@
class AudioSampleDataSource;
class AudioSampleBufferList;
+class BaseAudioMediaStreamTrackRendererUnit;
class CAAudioStreamDescription;
class AudioMediaStreamTrackRendererCocoa : public AudioMediaStreamTrackRenderer, public CanMakeWeakPtr<AudioMediaStreamTrackRendererCocoa, WeakPtrFactoryInitialization::Eager> {
WTF_MAKE_FAST_ALLOCATED;
public:
- AudioMediaStreamTrackRendererCocoa();
+ AudioMediaStreamTrackRendererCocoa(Init&&);
~AudioMediaStreamTrackRendererCocoa();
private:
@@ -59,6 +60,8 @@
void reset();
void setRegisteredDataSource(RefPtr<AudioSampleDataSource>&&);
+ BaseAudioMediaStreamTrackRendererUnit& rendererUnit();
+
std::unique_ptr<CAAudioStreamDescription> m_outputDescription;
RefPtr<AudioSampleDataSource> m_dataSource; // Used in background thread.
RefPtr<AudioSampleDataSource> m_registeredDataSource; // Used in main thread.
Modified: trunk/Source/WebCore/platform/mediastream/cocoa/AudioMediaStreamTrackRendererUnit.h (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/cocoa/AudioMediaStreamTrackRendererUnit.h 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/cocoa/AudioMediaStreamTrackRendererUnit.h 2021-10-29 13:36:03 UTC (rev 285027)
@@ -28,10 +28,9 @@
#if ENABLE(MEDIA_STREAM)
#include "AudioMediaStreamTrackRendererInternalUnit.h"
-#include <wtf/Forward.h>
+#include "BaseAudioMediaStreamTrackRendererUnit.h"
#include <wtf/HashSet.h>
#include <wtf/Lock.h>
-#include <wtf/Observer.h>
#include <wtf/UniqueRef.h>
#include <wtf/Vector.h>
#include <wtf/WeakHashSet.h>
@@ -43,7 +42,7 @@
class CAAudioStreamDescription;
class AudioMediaStreamTrackRendererInternalUnit;
-class AudioMediaStreamTrackRendererUnit {
+class AudioMediaStreamTrackRendererUnit : public BaseAudioMediaStreamTrackRendererUnit {
public:
WEBCORE_EXPORT static AudioMediaStreamTrackRendererUnit& singleton();
@@ -56,15 +55,14 @@
WEBCORE_EXPORT void render(size_t sampleCount, AudioBufferList&, uint64_t sampleTime, double hostTime, AudioUnitRenderActionFlags&);
void reset();
- void setAudioOutputDevice(const String&);
+ void retrieveFormatDescription(CompletionHandler<void(const CAAudioStreamDescription*)>&&);
- void addSource(Ref<AudioSampleDataSource>&&);
- void removeSource(AudioSampleDataSource&);
+ // BaseAudioMediaStreamTrackRendererUnit
+ void setAudioOutputDevice(const String&) final;
+ void addSource(Ref<AudioSampleDataSource>&&) final;
+ void removeSource(AudioSampleDataSource&) final;
+ void addResetObserver(ResetObserver& observer) final { m_resetObservers.add(observer); }
- using ResetObserver = Observer<void()>;
- void addResetObserver(ResetObserver& observer) { m_resetObservers.add(observer); }
- void retrieveFormatDescription(CompletionHandler<void(const CAAudioStreamDescription*)>&&);
-
private:
void start();
void stop();
Copied: trunk/Source/WebCore/platform/mediastream/cocoa/BaseAudioMediaStreamTrackRendererUnit.h (from rev 285026, trunk/Source/WebCore/platform/mediastream/AudioMediaStreamTrackRenderer.cpp) (0 => 285027)
--- trunk/Source/WebCore/platform/mediastream/cocoa/BaseAudioMediaStreamTrackRendererUnit.h (rev 0)
+++ trunk/Source/WebCore/platform/mediastream/cocoa/BaseAudioMediaStreamTrackRendererUnit.h 2021-10-29 13:36:03 UTC (rev 285027)
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if ENABLE(MEDIA_STREAM)
+
+#include <wtf/Forward.h>
+#include <wtf/Observer.h>
+
+namespace WebCore {
+
+class AudioSampleDataSource;
+
+class BaseAudioMediaStreamTrackRendererUnit {
+public:
+ virtual ~BaseAudioMediaStreamTrackRendererUnit() = default;
+
+ virtual void addSource(Ref<AudioSampleDataSource>&&) = 0;
+ virtual void removeSource(AudioSampleDataSource&) = 0;
+
+ virtual void setAudioOutputDevice(const String&) = 0;
+
+ using ResetObserver = Observer<void()>;
+ virtual void addResetObserver(ResetObserver&) = 0;
+};
+
+}
+
+#endif // ENABLE(MEDIA_STREAM)
Added: trunk/Source/WebCore/platform/mediastream/cocoa/IncomingAudioMediaStreamTrackRendererUnit.cpp (0 => 285027)
--- trunk/Source/WebCore/platform/mediastream/cocoa/IncomingAudioMediaStreamTrackRendererUnit.cpp (rev 0)
+++ trunk/Source/WebCore/platform/mediastream/cocoa/IncomingAudioMediaStreamTrackRendererUnit.cpp 2021-10-29 13:36:03 UTC (rev 285027)
@@ -0,0 +1,203 @@
+/*
+ * Copyright (C) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "IncomingAudioMediaStreamTrackRendererUnit.h"
+
+#if ENABLE(MEDIA_STREAM)
+
+#include "AudioMediaStreamTrackRendererUnit.h"
+#include "AudioSampleDataSource.h"
+#include "LibWebRTCAudioFormat.h"
+#include "LibWebRTCAudioModule.h"
+#include "Logging.h"
+#include "WebAudioBufferList.h"
+#include <pal/avfoundation/MediaTimeAVFoundation.h>
+#include <wtf/WorkQueue.h>
+
+#include <pal/cf/CoreMediaSoftLink.h>
+
+namespace WebCore {
+
+IncomingAudioMediaStreamTrackRendererUnit::IncomingAudioMediaStreamTrackRendererUnit(LibWebRTCAudioModule& audioModule)
+ : m_audioModule(audioModule)
+ , m_queue(WorkQueue::create("IncomingAudioMediaStreamTrackRendererUnit", WorkQueue::QOS::UserInitiated))
+#if !RELEASE_LOG_DISABLED
+ , m_logIdentifier(LoggerHelper::uniqueLogIdentifier())
+#endif
+{
+}
+
+IncomingAudioMediaStreamTrackRendererUnit::~IncomingAudioMediaStreamTrackRendererUnit()
+{
+}
+
+void IncomingAudioMediaStreamTrackRendererUnit::setAudioOutputDevice(const String& deviceID)
+{
+ AudioMediaStreamTrackRendererUnit::singleton().setAudioOutputDevice(deviceID);
+}
+
+void IncomingAudioMediaStreamTrackRendererUnit::addResetObserver(ResetObserver& observer)
+{
+ AudioMediaStreamTrackRendererUnit::singleton().addResetObserver(observer);
+}
+
+void IncomingAudioMediaStreamTrackRendererUnit::addSource(Ref<AudioSampleDataSource>&& source)
+{
+#if !RELEASE_LOG_DISABLED
+ source->logger().logAlways(LogWebRTC, "IncomingAudioMediaStreamTrackRendererUnit::addSource ", source->logIdentifier());
+#endif
+ ASSERT(isMainThread());
+#if !RELEASE_LOG_DISABLED
+ if (!m_logger)
+ m_logger = &source->logger();
+#endif
+
+ ASSERT(source->outputDescription());
+ if (!source->outputDescription())
+ return;
+ auto outputDescription = *source->outputDescription();
+
+ ASSERT(!m_sources.contains(source.get()));
+ bool shouldStart = m_sources.isEmpty();
+ m_sources.add(WTFMove(source));
+
+ postTask([this, newSources = copyToVector(m_sources), shouldStart, outputDescription]() mutable {
+ m_renderSources = WTFMove(newSources);
+ if (!shouldStart)
+ return;
+
+ m_outputStreamDescription = outputDescription;
+ m_audioBufferList = makeUnique<WebAudioBufferList>(m_outputStreamDescription);
+ m_sampleCount = m_outputStreamDescription.sampleRate() / 100;
+ m_audioBufferList->setSampleCount(m_sampleCount);
+ m_mixedSource = AudioSampleDataSource::create(m_outputStreamDescription.sampleRate() * 0.5, *this, LibWebRTCAudioModule::PollSamplesCount + 1);
+ m_mixedSource->setInputFormat(outputDescription);
+ m_mixedSource->setOutputFormat(outputDescription);
+ m_writeCount = 0;
+ callOnMainThread([this, protectedModule = Ref { m_audioModule }, source = m_mixedSource]() mutable {
+ m_registeredMixedSource = WTFMove(source);
+ start();
+ });
+ });
+}
+
+void IncomingAudioMediaStreamTrackRendererUnit::removeSource(AudioSampleDataSource& source)
+{
+#if !RELEASE_LOG_DISABLED
+ source.logger().logAlways(LogWebRTC, "IncomingAudioMediaStreamTrackRendererUnit::removeSource ", source.logIdentifier());
+#endif
+ ASSERT(isMainThread());
+
+ bool shouldStop = !m_sources.isEmpty();
+ m_sources.remove(source);
+ shouldStop &= m_sources.isEmpty();
+
+ postTask([this, newSources = copyToVector(m_sources), shouldStop]() mutable {
+ m_renderSources = WTFMove(newSources);
+ if (!shouldStop)
+ return;
+
+ callOnMainThread([this, protectedModule = Ref { m_audioModule }] {
+ stop();
+ });
+ });
+
+}
+
+void IncomingAudioMediaStreamTrackRendererUnit::start()
+{
+ RELEASE_LOG(WebRTC, "IncomingAudioMediaStreamTrackRendererUnit::start");
+ ASSERT(isMainThread());
+
+ AudioMediaStreamTrackRendererUnit::singleton().addSource(*m_registeredMixedSource);
+ m_audioModule.startIncomingAudioRendering();
+}
+
+void IncomingAudioMediaStreamTrackRendererUnit::stop()
+{
+ RELEASE_LOG(WebRTC, "IncomingAudioMediaStreamTrackRendererUnit::stop");
+ ASSERT(isMainThread());
+
+ AudioMediaStreamTrackRendererUnit::singleton().removeSource(*m_registeredMixedSource);
+ m_registeredMixedSource = nullptr;
+ m_audioModule.stopIncomingAudioRendering();
+}
+
+void IncomingAudioMediaStreamTrackRendererUnit::postTask(Function<void()>&& callback)
+{
+ // We protect 'this' by taking a ref to its audio module.
+ m_queue->dispatch([protectedModule = Ref { m_audioModule }, callback = WTFMove(callback)]() mutable {
+ callback();
+ });
+}
+
+void IncomingAudioMediaStreamTrackRendererUnit::newAudioChunkPushed()
+{
+ DisableMallocRestrictionsForCurrentThreadScope disableMallocRestrictions;
+ postTask([this, currentAudioSampleCount = m_audioModule.currentAudioSampleCount()] {
+ renderAudioChunk(currentAudioSampleCount);
+ });
+}
+
+void IncomingAudioMediaStreamTrackRendererUnit::renderAudioChunk(uint64_t currentAudioSampleCount)
+{
+ ASSERT(!isMainThread());
+
+ uint64_t timeStamp = currentAudioSampleCount * m_outputStreamDescription.sampleRate() / LibWebRTCAudioFormat::sampleRate;
+
+ // Mix all sources.
+ bool hasCopiedData = false;
+ for (auto& source : m_renderSources) {
+ if (source->pullAvailableSampleChunk(*m_audioBufferList, m_sampleCount, timeStamp, hasCopiedData ? AudioSampleDataSource::Mix : AudioSampleDataSource::Copy))
+ hasCopiedData = true;
+ }
+
+ CMTime startTime = PAL::CMTimeMake(m_writeCount, m_outputStreamDescription.sampleRate());
+ if (hasCopiedData)
+ m_mixedSource->pushSamples(PAL::toMediaTime(startTime), *m_audioBufferList, m_sampleCount);
+ m_writeCount += m_sampleCount;
+}
+
+#if !RELEASE_LOG_DISABLED
+const Logger& IncomingAudioMediaStreamTrackRendererUnit::logger() const
+{
+ return *m_logger;
+}
+
+WTFLogChannel& IncomingAudioMediaStreamTrackRendererUnit::logChannel() const
+{
+ return LogWebRTC;
+}
+
+const void* IncomingAudioMediaStreamTrackRendererUnit::logIdentifier() const
+{
+ return m_logIdentifier;
+}
+#endif
+
+} // namespace WebCore
+
+#endif // ENABLE(MEDIA_STREAM)
Added: trunk/Source/WebCore/platform/mediastream/cocoa/IncomingAudioMediaStreamTrackRendererUnit.h (0 => 285027)
--- trunk/Source/WebCore/platform/mediastream/cocoa/IncomingAudioMediaStreamTrackRendererUnit.h (rev 0)
+++ trunk/Source/WebCore/platform/mediastream/cocoa/IncomingAudioMediaStreamTrackRendererUnit.h 2021-10-29 13:36:03 UTC (rev 285027)
@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if ENABLE(MEDIA_STREAM)
+
+#include "BaseAudioMediaStreamTrackRendererUnit.h"
+#include "CAAudioStreamDescription.h"
+#include <wtf/FastMalloc.h>
+#include <wtf/Forward.h>
+#include <wtf/HashSet.h>
+#include <wtf/LoggerHelper.h>
+#include <wtf/Vector.h>
+
+namespace WTF {
+class WorkQueue;
+}
+
+namespace WebCore {
+
+class AudioMediaStreamTrackRendererInternalUnit;
+class AudioSampleDataSource;
+class AudioSampleBufferList;
+class CAAudioStreamDescription;
+class LibWebRTCAudioModule;
+class WebAudioBufferList;
+
+class IncomingAudioMediaStreamTrackRendererUnit : public BaseAudioMediaStreamTrackRendererUnit
+#if !RELEASE_LOG_DISABLED
+ , public LoggerHelper
+#endif
+{
+ WTF_MAKE_FAST_ALLOCATED;
+public:
+ explicit IncomingAudioMediaStreamTrackRendererUnit(LibWebRTCAudioModule&);
+ ~IncomingAudioMediaStreamTrackRendererUnit();
+
+ void newAudioChunkPushed();
+
+private:
+ void start();
+ void stop();
+ void postTask(Function<void()>&&);
+ void renderAudioChunk(uint64_t currentAudioSampleCount);
+
+ // BaseAudioMediaStreamTrackRendererUnit
+ void setAudioOutputDevice(const String&) final;
+ void addResetObserver(ResetObserver&) final;
+ void addSource(Ref<AudioSampleDataSource>&&) final;
+ void removeSource(AudioSampleDataSource&) final;
+
+#if !RELEASE_LOG_DISABLED
+ // LoggerHelper.
+ const Logger& logger() const final;
+ const char* logClassName() const final { return "IncomingAudioMediaStreamTrackRendererUnit"; }
+ WTFLogChannel& logChannel() const final;
+ const void* logIdentifier() const final;
+#endif
+
+ // Main thread variables.
+ LibWebRTCAudioModule& m_audioModule;
+ Ref<WTF::WorkQueue> m_queue;
+ HashSet<Ref<AudioSampleDataSource>> m_sources;
+ RefPtr<AudioSampleDataSource> m_registeredMixedSource;
+
+ // Background thread variables.
+ Vector<Ref<AudioSampleDataSource>> m_renderSources;
+ RefPtr<AudioSampleDataSource> m_mixedSource;
+ CAAudioStreamDescription m_outputStreamDescription;
+ std::unique_ptr<WebAudioBufferList> m_audioBufferList;
+ size_t m_sampleCount { 0 };
+ size_t m_writeCount { 0 };
+
+#if !RELEASE_LOG_DISABLED
+ RefPtr<const Logger> m_logger;
+ const void* m_logIdentifier;
+#endif
+};
+
+}
+
+#endif // ENABLE(MEDIA_STREAM)
Modified: trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.cpp (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.cpp 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.cpp 2021-10-29 13:36:03 UTC (rev 285027)
@@ -32,6 +32,10 @@
#include "Logging.h"
#include <wtf/FastMalloc.h>
+#if PLATFORM(COCOA)
+#include "IncomingAudioMediaStreamTrackRendererUnit.h"
+#endif
+
namespace WebCore {
LibWebRTCAudioModule::LibWebRTCAudioModule()
@@ -40,6 +44,10 @@
{
}
+LibWebRTCAudioModule::~LibWebRTCAudioModule()
+{
+}
+
int32_t LibWebRTCAudioModule::RegisterAudioCallback(webrtc::AudioTransport* audioTransport)
{
RELEASE_LOG(WebRTC, "LibWebRTCAudioModule::RegisterAudioCallback %d", !!audioTransport);
@@ -62,6 +70,9 @@
m_queue->dispatch([this, protectedThis = rtc::scoped_refptr<webrtc::AudioDeviceModule>(this)] {
m_pollingTime = MonotonicTime::now();
+#if PLATFORM(COCOA)
+ m_currentAudioSampleCount = 0;
+#endif
pollAudioData();
});
return 0;
@@ -129,9 +140,23 @@
int64_t ntpTime = -1;
char data[LibWebRTCAudioFormat::sampleByteSize * channels * LibWebRTCAudioFormat::chunkSampleCount];
m_audioTransport->PullRenderData(LibWebRTCAudioFormat::sampleByteSize * 8, LibWebRTCAudioFormat::sampleRate, channels, LibWebRTCAudioFormat::chunkSampleCount, data, &elapsedTime, &ntpTime);
+#if PLATFORM(COCOA)
+ if (m_isRenderingIncomingAudio)
+ m_incomingAudioMediaStreamTrackRendererUnit->newAudioChunkPushed();
+ m_currentAudioSampleCount += LibWebRTCAudioFormat::chunkSampleCount;
+#endif
}
}
+#if PLATFORM(COCOA)
+BaseAudioMediaStreamTrackRendererUnit& LibWebRTCAudioModule::incomingAudioMediaStreamTrackRendererUnit()
+{
+ if (!m_incomingAudioMediaStreamTrackRendererUnit)
+ m_incomingAudioMediaStreamTrackRendererUnit = makeUnique<IncomingAudioMediaStreamTrackRendererUnit>(*this);
+ return *m_incomingAudioMediaStreamTrackRendererUnit;
+}
+#endif
+
} // namespace WebCore
#endif // USE(LIBWEBRTC)
Modified: trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.h (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.h 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.h 2021-10-29 13:36:03 UTC (rev 285027)
@@ -39,6 +39,8 @@
ALLOW_UNUSED_PARAMETERS_END
namespace WebCore {
+class BaseAudioMediaStreamTrackRendererUnit;
+class IncomingAudioMediaStreamTrackRendererUnit;
// LibWebRTCAudioModule is pulling streamed data to ensure audio data is passed to the audio track.
class LibWebRTCAudioModule : public webrtc::AudioDeviceModule {
@@ -45,9 +47,19 @@
WTF_MAKE_FAST_ALLOCATED;
public:
LibWebRTCAudioModule();
+ ~LibWebRTCAudioModule();
static constexpr unsigned PollSamplesCount = 1;
+ void ref() { AddRef(); }
+ void deref() { Release(); }
+#if PLATFORM(COCOA)
+ void startIncomingAudioRendering() { m_isRenderingIncomingAudio = true; }
+ void stopIncomingAudioRendering() { m_isRenderingIncomingAudio = false; }
+ BaseAudioMediaStreamTrackRendererUnit& incomingAudioMediaStreamTrackRendererUnit();
+ uint64_t currentAudioSampleCount() const { return m_currentAudioSampleCount; }
+#endif
+
private:
template<typename U> U shouldNotBeCalled(U value) const
{
@@ -135,6 +147,12 @@
MonotonicTime m_pollingTime;
Timer m_logTimer;
int m_timeSpent { 0 };
+
+#if PLATFORM(COCOA)
+ uint64_t m_currentAudioSampleCount { 0 };
+ bool m_isRenderingIncomingAudio { false };
+ std::unique_ptr<IncomingAudioMediaStreamTrackRendererUnit> m_incomingAudioMediaStreamTrackRendererUnit;
+#endif
};
} // namespace WebCore
Modified: trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCProvider.cpp (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCProvider.cpp 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCProvider.cpp 2021-10-29 13:36:03 UTC (rev 285027)
@@ -70,6 +70,16 @@
}
#endif
+#if USE(LIBWEBRTC)
+LibWebRTCProvider::LibWebRTCProvider()
+{
+}
+#endif
+
+LibWebRTCProvider::~LibWebRTCProvider()
+{
+}
+
#if !USE(LIBWEBRTC) || !PLATFORM(COCOA)
void LibWebRTCProvider::registerWebKitVP9Decoder()
{
@@ -271,9 +281,17 @@
return m_factory;
}
+void LibWebRTCProvider::clearFactory()
+{
+ m_audioModule = nullptr;
+ m_factory = nullptr;
+}
+
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> LibWebRTCProvider::createPeerConnectionFactory(rtc::Thread* networkThread, rtc::Thread* signalingThread)
{
- auto audioModule = rtc::scoped_refptr<webrtc::AudioDeviceModule>(new rtc::RefCountedObject<LibWebRTCAudioModule>());
+ ASSERT(!m_audioModule);
+ auto audioModule = rtc::scoped_refptr<LibWebRTCAudioModule>(new rtc::RefCountedObject<LibWebRTCAudioModule>());
+ m_audioModule = audioModule.get();
return webrtc::CreatePeerConnectionFactory(networkThread, signalingThread, signalingThread, WTFMove(audioModule), webrtc::CreateBuiltinAudioEncoderFactory(), webrtc::CreateBuiltinAudioDecoderFactory(), createEncoderFactory(), createDecoderFactory(), nullptr, nullptr, nullptr);
}
Modified: trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCProvider.h (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCProvider.h 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCProvider.h 2021-10-29 13:36:03 UTC (rev 285027)
@@ -79,7 +79,7 @@
public:
static UniqueRef<LibWebRTCProvider> create();
- virtual ~LibWebRTCProvider() = default;
+ virtual ~LibWebRTCProvider();
static bool webRTCAvailable();
static void registerWebKitVP9Decoder();
@@ -103,6 +103,7 @@
virtual rtc::scoped_refptr<webrtc::PeerConnectionInterface> createPeerConnection(DocumentIdentifier, webrtc::PeerConnectionObserver&, rtc::PacketSocketFactory*, webrtc::PeerConnectionInterface::RTCConfiguration&&);
webrtc::PeerConnectionFactoryInterface* factory();
+ LibWebRTCAudioModule* audioModule();
// FIXME: Make these methods not static.
static void callOnWebRTCNetworkThread(Function<void()>&&);
@@ -134,7 +135,7 @@
std::optional<RTCRtpCapabilities> receiverCapabilities(const String& kind);
std::optional<RTCRtpCapabilities> senderCapabilities(const String& kind);
- void clearFactory() { m_factory = nullptr; }
+ void clearFactory();
virtual void setLoggingLevel(WTFLogLevel);
void setEnableWebRTCEncryption(bool);
@@ -150,7 +151,7 @@
virtual std::unique_ptr<SuspendableSocketFactory> createSocketFactory(String&& /* userAgent */, bool /* isFirstParty */, RegistrableDomain&&) { return nullptr; }
protected:
- LibWebRTCProvider() = default;
+ LibWebRTCProvider();
rtc::scoped_refptr<webrtc::PeerConnectionInterface> createPeerConnection(webrtc::PeerConnectionObserver&, rtc::NetworkManager&, rtc::PacketSocketFactory&, webrtc::PeerConnectionInterface::RTCConfiguration&&, std::unique_ptr<webrtc::AsyncResolverFactory>&&);
@@ -171,6 +172,7 @@
// FIXME: Remove m_useNetworkThreadWithSocketServer member variable and make it a global.
bool m_useNetworkThreadWithSocketServer { true };
+ RefPtr<LibWebRTCAudioModule> m_audioModule;
rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface> m_factory;
bool m_disableNonLocalhostConnections { false };
bool m_supportsH265 { false };
@@ -187,4 +189,11 @@
#endif
};
+#if USE(LIBWEBRTC)
+inline LibWebRTCAudioModule* LibWebRTCProvider::audioModule()
+{
+ return m_audioModule.get();
+}
+#endif
+
} // namespace WebCore
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingAudioSourceCocoa.cpp (285026 => 285027)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingAudioSourceCocoa.cpp 2021-10-29 13:04:17 UTC (rev 285026)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingAudioSourceCocoa.cpp 2021-10-29 13:36:03 UTC (rev 285027)
@@ -132,7 +132,7 @@
m_numberOfFrames = 0;
}
- CMTime startTime = PAL::CMTimeMake(m_numberOfFrames, sampleRate);
+ CMTime startTime = PAL::CMTimeMake(audioModule() ? audioModule()->currentAudioSampleCount() : m_numberOfFrames, LibWebRTCAudioFormat::sampleRate);
auto mediaTime = PAL::toMediaTime(startTime);
m_numberOfFrames += numberOfFrames;
