Diff
Modified: trunk/Source/WebCore/ChangeLog (268576 => 268577)
--- trunk/Source/WebCore/ChangeLog 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebCore/ChangeLog 2020-10-16 09:48:57 UTC (rev 268577)
@@ -1,3 +1,32 @@
+2020-10-16 Youenn Fablet <[email protected]>
+
+ Add support for GPUProcess WebAudio media element providers
+ https://bugs.webkit.org/show_bug.cgi?id=217704
+
+ Reviewed by Eric Carlson.
+
+ Update AudioSourceProviderAVFObjC so that a specific ring buffer can be provided and to be notified of new samples being pushed.
+
+ Covered by existing web audio tests run with GPUProcess.
+
+ * WebCore.xcodeproj/project.pbxproj:
+ * platform/audio/AudioSourceProvider.h:
+ (WebCore::AudioSourceProvider::isHandlingAVPlayer const):
+ * platform/audio/cocoa/AudioSampleDataSource.mm:
+ (WebCore::AudioSampleDataSource::pullSamplesInternal):
+ When we are too close to the end of available data, do as if we are paused and keep the same time entry for the next read operation.
+ This will basically make us wait for the next push of data before restarting to read succesfully.
+ * platform/graphics/avfoundation/AudioSourceProviderAVFObjC.h:
+ (isType):
+ * platform/graphics/avfoundation/AudioSourceProviderAVFObjC.mm:
+ (WebCore::AudioSourceProviderAVFObjC::prepare):
+ (WebCore::AudioSourceProviderAVFObjC::process):
+ (WebCore::AudioSourceProviderAVFObjC::setAudioCallback):
+ (WebCore::AudioSourceProviderAVFObjC::setRingBufferCreationCallback):
+ * platform/mediastream/mac/WebAudioSourceProviderCocoa.mm:
+ (WebCore::WebAudioSourceProviderCocoa::receivedNewAudioSamples):
+ We should use the input sample rate to compute the input media time.
+
2020-10-16 Philippe Normand <[email protected]>
[GStreamer] Encoder probing support for the registry scanner
Modified: trunk/Source/WebCore/Headers.cmake (268576 => 268577)
--- trunk/Source/WebCore/Headers.cmake 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebCore/Headers.cmake 2020-10-16 09:48:57 UTC (rev 268577)
@@ -1067,6 +1067,7 @@
platform/audio/AudioIOCallback.h
platform/audio/AudioSession.h
platform/audio/AudioSourceProvider.h
+ platform/audio/AudioSourceProviderClient.h
platform/audio/AudioStreamDescription.h
platform/audio/AudioUtilities.h
platform/audio/NowPlayingInfo.h
Modified: trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj (268576 => 268577)
--- trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2020-10-16 09:48:57 UTC (rev 268577)
@@ -4197,7 +4197,7 @@
CD7E05221651C28200C1201F /* WebCoreAVFResourceLoader.mm in Sources */ = {isa = PBXBuildFile; fileRef = CD7E05211651A84100C1201F /* WebCoreAVFResourceLoader.mm */; };
CD83D36221122A210076E11C /* DeferrableTask.h in Headers */ = {isa = PBXBuildFile; fileRef = CD83D35A211110820076E11C /* DeferrableTask.h */; settings = {ATTRIBUTES = (Private, ); }; };
CD8A7BBB197735FE00CBD643 /* AudioSourceProviderAVFObjC.mm in Sources */ = {isa = PBXBuildFile; fileRef = CD8A7BB9197735FE00CBD643 /* AudioSourceProviderAVFObjC.mm */; };
- CD8A7BBC197735FE00CBD643 /* AudioSourceProviderAVFObjC.h in Headers */ = {isa = PBXBuildFile; fileRef = CD8A7BBA197735FE00CBD643 /* AudioSourceProviderAVFObjC.h */; };
+ CD8A7BBC197735FE00CBD643 /* AudioSourceProviderAVFObjC.h in Headers */ = {isa = PBXBuildFile; fileRef = CD8A7BBA197735FE00CBD643 /* AudioSourceProviderAVFObjC.h */; settings = {ATTRIBUTES = (Private, ); }; };
CD8ACA891D237AA200ECC59E /* RemoteCommandListenerMac.h in Headers */ = {isa = PBXBuildFile; fileRef = CD8ACA871D237AA200ECC59E /* RemoteCommandListenerMac.h */; };
CD8ACA8F1D23971900ECC59E /* MediaRemoteSoftLink.h in Headers */ = {isa = PBXBuildFile; fileRef = CD8ACA8D1D23971900ECC59E /* MediaRemoteSoftLink.h */; };
CD8B5A42180D149A008B8E65 /* VideoTrackPrivateMediaSourceAVFObjC.mm in Sources */ = {isa = PBXBuildFile; fileRef = CD8B5A40180D149A008B8E65 /* VideoTrackPrivateMediaSourceAVFObjC.mm */; };
Modified: trunk/Source/WebCore/platform/audio/AudioSourceProvider.h (268576 => 268577)
--- trunk/Source/WebCore/platform/audio/AudioSourceProvider.h 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebCore/platform/audio/AudioSourceProvider.h 2020-10-16 09:48:57 UTC (rev 268577)
@@ -43,6 +43,8 @@
// If a client is set, we call it back when the audio format is available or changes.
virtual void setClient(AudioSourceProviderClient*) { };
+ virtual bool isHandlingAVPlayer() const { return false; }
+
virtual ~AudioSourceProvider() = default;
};
Modified: trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm (268576 => 268577)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm 2020-10-16 09:48:57 UTC (rev 268577)
@@ -260,9 +260,8 @@
if (timeStamp >= endFrame)
m_endFrameWhenNotEnoughData = endFrame;
} else {
- // We are too close from endFrame, let's back up a little bit.
- uint64_t framesAvailable = endFrame - timeStamp;
- m_outputSampleOffset -= sampleCount - framesAvailable;
+ // We are too close from endFrame, let's wait for more data to be pushed.
+ m_outputSampleOffset -= sampleCount;
dispatch_async(dispatch_get_main_queue(), [logIdentifier = LOGIDENTIFIER, outputSampleOffset = m_outputSampleOffset, this, protectedThis = makeRefPtr(*this)] {
ALWAYS_LOG(logIdentifier, "updating offset to ", outputSampleOffset);
});
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/AudioSourceProviderAVFObjC.h (268576 => 268577)
--- trunk/Source/WebCore/platform/graphics/avfoundation/AudioSourceProviderAVFObjC.h 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/AudioSourceProviderAVFObjC.h 2020-10-16 09:48:57 UTC (rev 268577)
@@ -23,8 +23,7 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-#ifndef AudioSourceProviderAVFObjC_h
-#define AudioSourceProviderAVFObjC_h
+#pragma once
#if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
@@ -33,6 +32,8 @@
#include <wtf/MediaTime.h>
#include <wtf/RetainPtr.h>
#include <wtf/ThreadSafeRefCounted.h>
+#include <wtf/TypeCasts.h>
+#include <wtf/UniqueRef.h>
OBJC_CLASS AVAssetTrack;
OBJC_CLASS AVPlayerItem;
@@ -47,7 +48,10 @@
namespace WebCore {
+class AudioStreamDescription;
+class CAAudioStreamDescription;
class CARingBuffer;
+class PlatformAudioData;
class AudioSourceProviderAVFObjC : public ThreadSafeRefCounted<AudioSourceProviderAVFObjC>, public AudioSourceProvider {
public:
@@ -57,6 +61,11 @@
void setPlayerItem(AVPlayerItem *);
void setAudioTrack(AVAssetTrack *);
+ using AudioCallback = Function<void(uint64_t startFrame, uint64_t numberOfFrames)>;
+ WEBCORE_EXPORT void setAudioCallback(AudioCallback&&);
+ using RingBufferCreationCallback = Function<UniqueRef<CARingBuffer>(const CAAudioStreamDescription&, size_t)>;
+ WEBCORE_EXPORT void setRingBufferCreationCallback(RingBufferCreationCallback&&);
+
private:
AudioSourceProviderAVFObjC(AVPlayerItem *);
@@ -66,6 +75,7 @@
// AudioSourceProvider
void provideInput(AudioBus*, size_t framesToProcess) override;
void setClient(AudioSourceProviderClient*) override;
+ bool isHandlingAVPlayer() const final { return true; }
static void initCallback(MTAudioProcessingTapRef, void*, void**);
static void finalizeCallback(MTAudioProcessingTapRef);
@@ -100,10 +110,14 @@
class TapStorage;
RefPtr<TapStorage> m_tapStorage;
+ AudioCallback m_audioCallback;
+ RingBufferCreationCallback m_ringBufferCallback;
};
-
+
}
-#endif
+SPECIALIZE_TYPE_TRAITS_BEGIN(WebCore::AudioSourceProviderAVFObjC)
+ static bool isType(const WebCore::AudioSourceProvider& provider) { return provider.isHandlingAVPlayer(); }
+SPECIALIZE_TYPE_TRAITS_END()
-#endif
+#endif // ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/AudioSourceProviderAVFObjC.mm (268576 => 268577)
--- trunk/Source/WebCore/platform/graphics/avfoundation/AudioSourceProviderAVFObjC.mm 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/AudioSourceProviderAVFObjC.mm 2020-10-16 09:48:57 UTC (rev 268577)
@@ -337,8 +337,13 @@
// Make the ringbuffer large enough to store at least two callbacks worth of audio, or 1s, whichever is larger.
size_t capacity = std::max(static_cast<size_t>(2 * maxFrames), static_cast<size_t>(kRingBufferDuration * sampleRate));
- m_ringBuffer = makeUnique<CARingBuffer>();
- m_ringBuffer->allocate(CAAudioStreamDescription(*processingFormat), capacity);
+ CAAudioStreamDescription description { *processingFormat };
+ if (m_ringBufferCallback)
+ m_ringBuffer = m_ringBufferCallback(description, capacity).moveToUniquePtr();
+ else {
+ m_ringBuffer = makeUnique<CARingBuffer>();
+ m_ringBuffer->allocate(description, capacity);
+ }
// AudioBufferList is a variable-length struct, so create on the heap with a generic new() operator
// with a custom size, and initialize the struct manually.
@@ -419,8 +424,23 @@
memset(buffer.mData, 0, buffer.mDataByteSize);
}
*numberFramesOut = 0;
+
+ if (m_audioCallback)
+ m_audioCallback(endFrame, itemCount);
}
+void AudioSourceProviderAVFObjC::setAudioCallback(AudioCallback&& callback)
+{
+ ASSERT(!m_avAudioMix);
+ m_audioCallback = WTFMove(callback);
}
+void AudioSourceProviderAVFObjC::setRingBufferCreationCallback(RingBufferCreationCallback&& callback)
+{
+ ASSERT(!m_avAudioMix);
+ m_ringBufferCallback = WTFMove(callback);
+}
+
+}
+
#endif // ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
Modified: trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderCocoa.mm (268576 => 268577)
--- trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderCocoa.mm 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderCocoa.mm 2020-10-16 09:48:57 UTC (rev 268577)
@@ -141,7 +141,7 @@
if (!m_dataSource)
return;
- m_dataSource->pushSamples(MediaTime(m_writeCount, m_outputDescription->sampleRate()), data, frameCount);
+ m_dataSource->pushSamples(MediaTime(m_writeCount, m_inputDescription->sampleRate()), data, frameCount);
m_writeCount += frameCount;
}
Modified: trunk/Source/WebKit/CMakeLists.txt (268576 => 268577)
--- trunk/Source/WebKit/CMakeLists.txt 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/CMakeLists.txt 2020-10-16 09:48:57 UTC (rev 268577)
@@ -219,6 +219,7 @@
WebProcess/GPU/media/MediaPlayerPrivateRemote
WebProcess/GPU/media/RemoteAudioDestinationProxy
WebProcess/GPU/media/RemoteAudioSession
+ WebProcess/GPU/media/RemoteAudioSourceProviderManager
WebProcess/GPU/media/RemoteCDMInstanceSession
WebProcess/GPU/media/RemoteLegacyCDMSession
Modified: trunk/Source/WebKit/ChangeLog (268576 => 268577)
--- trunk/Source/WebKit/ChangeLog 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/ChangeLog 2020-10-16 09:48:57 UTC (rev 268577)
@@ -1,3 +1,60 @@
+2020-10-16 Youenn Fablet <[email protected]>
+
+ Add support for GPUProcess WebAudio media element providers
+ https://bugs.webkit.org/show_bug.cgi?id=217704
+
+ Reviewed by Eric Carlson.
+
+ Introduce RemoteAudioSourceProviderManager which receives IPC messages from GPU process for web audio data.
+ RemoteAudioSourceProviderManager sends it to the identified RemoteAudioSourceProvider.
+
+ Introduce RemoteAudioSourceProviderProxy which is a client to the AV AudioSourceProviderAVFObjC.
+ RemoteAudioSourceProviderProxy will send IPC messages to RemoteAudioSourceProviderManager to send audio data and description.
+
+ * CMakeLists.txt:
+ * DerivedSources-input.xcfilelist:
+ * DerivedSources-output.xcfilelist:
+ * DerivedSources.make:
+ * GPUProcess/media/RemoteAudioSourceProviderProxy.cpp: Added.
+ (WebKit::RemoteAudioSourceProviderProxy::create):
+ (WebKit::RemoteAudioSourceProviderProxy::RemoteAudioSourceProviderProxy):
+ (WebKit::RemoteAudioSourceProviderProxy::createRingBuffer):
+ (WebKit::RemoteAudioSourceProviderProxy::newAudioSamples):
+ (WebKit::RemoteAudioSourceProviderProxy::storageChanged):
+ * GPUProcess/media/RemoteAudioSourceProviderProxy.h: Added.
+ * GPUProcess/media/RemoteMediaPlayerProxy.cpp:
+ (WebKit::RemoteMediaPlayerProxy::~RemoteMediaPlayerProxy):
+ (WebKit::RemoteMediaPlayerProxy::createAudioSourceProvider):
+ (WebKit::RemoteMediaPlayerProxy::setShouldEnableAudioSourceProvider):
+ * GPUProcess/media/RemoteMediaPlayerProxy.h:
+ * GPUProcess/media/RemoteMediaPlayerProxy.messages.in:
+ * SourcesCocoa.txt:
+ * WebKit.xcodeproj/project.pbxproj:
+ * WebProcess/GPU/GPUProcessConnection.cpp:
+ (WebKit::GPUProcessConnection::audioSourceProviderManager):
+ * WebProcess/GPU/GPUProcessConnection.h:
+ * WebProcess/GPU/media/RemoteAudioSourceProvider.cpp:
+ (WebKit::RemoteAudioSourceProvider::create):
+ (WebKit::RemoteAudioSourceProvider::RemoteAudioSourceProvider):
+ (WebKit::RemoteAudioSourceProvider::close):
+ (WebKit::RemoteAudioSourceProvider::hasNewClient):
+ * WebProcess/GPU/media/RemoteAudioSourceProviderManager.cpp: Added.
+ (WebKit::RemoteAudioSourceProviderManager::RemoteAudioSourceProviderManager):
+ (WebKit::RemoteAudioSourceProviderManager::~RemoteAudioSourceProviderManager):
+ (WebKit::RemoteAudioSourceProviderManager::setConnection):
+ (WebKit::RemoteAudioSourceProviderManager::addProvider):
+ (WebKit::RemoteAudioSourceProviderManager::removeProvider):
+ (WebKit::RemoteAudioSourceProviderManager::dispatchToThread):
+ (WebKit::RemoteAudioSourceProviderManager::audioStorageChanged):
+ (WebKit::RemoteAudioSourceProviderManager::audioSamplesAvailable):
+ (WebKit::RemoteAudioSourceProviderManager::RemoteAudio::RemoteAudio):
+ (WebKit::RemoteAudioSourceProviderManager::RemoteAudio::setStorage):
+ (WebKit::RemoteAudioSourceProviderManager::RemoteAudio::audioSamplesAvailable):
+ * WebProcess/GPU/media/RemoteAudioSourceProviderManager.h: Added.
+ (WebKit::RemoteAudioSourceProviderManager::create):
+ * WebProcess/GPU/media/RemoteAudioSourceProviderManager.messages.in: Added.
+ * WebProcess/cocoa/RemoteCaptureSampleManager.cpp:
+
2020-10-15 Jiewen Tan <[email protected]>
[WebAuthn] Move AppAttestInternal related code from WKA to OpenSource
Modified: trunk/Source/WebKit/DerivedSources-input.xcfilelist (268576 => 268577)
--- trunk/Source/WebKit/DerivedSources-input.xcfilelist 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/DerivedSources-input.xcfilelist 2020-10-16 09:48:57 UTC (rev 268577)
@@ -127,6 +127,7 @@
$(PROJECT_DIR)/WebProcess/GPU/media/MediaPlayerPrivateRemote.messages.in
$(PROJECT_DIR)/WebProcess/GPU/media/RemoteAudioDestinationProxy.messages.in
$(PROJECT_DIR)/WebProcess/GPU/media/RemoteAudioSession.messages.in
+$(PROJECT_DIR)/WebProcess/GPU/media/RemoteAudioSourceProviderManager.messages.in
$(PROJECT_DIR)/WebProcess/GPU/media/RemoteCDMInstanceSession.messages.in
$(PROJECT_DIR)/WebProcess/GPU/media/RemoteLegacyCDMSession.messages.in
$(PROJECT_DIR)/WebProcess/GPU/media/ios/RemoteMediaSessionHelper.messages.in
Modified: trunk/Source/WebKit/DerivedSources-output.xcfilelist (268576 => 268577)
--- trunk/Source/WebKit/DerivedSources-output.xcfilelist 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/DerivedSources-output.xcfilelist 2020-10-16 09:48:57 UTC (rev 268577)
@@ -154,6 +154,9 @@
$(BUILT_PRODUCTS_DIR)/DerivedSources/WebKit2/RemoteAudioSessionProxyMessageReceiver.cpp
$(BUILT_PRODUCTS_DIR)/DerivedSources/WebKit2/RemoteAudioSessionProxyMessages.h
$(BUILT_PRODUCTS_DIR)/DerivedSources/WebKit2/RemoteAudioSessionProxyMessagesReplies.h
+$(BUILT_PRODUCTS_DIR)/DerivedSources/WebKit2/RemoteAudioSourceProviderManagerMessageReceiver.cpp
+$(BUILT_PRODUCTS_DIR)/DerivedSources/WebKit2/RemoteAudioSourceProviderManagerMessages.h
+$(BUILT_PRODUCTS_DIR)/DerivedSources/WebKit2/RemoteAudioSourceProviderManagerMessagesReplies.h
$(BUILT_PRODUCTS_DIR)/DerivedSources/WebKit2/RemoteCDMFactoryProxyMessageReceiver.cpp
$(BUILT_PRODUCTS_DIR)/DerivedSources/WebKit2/RemoteCDMFactoryProxyMessages.h
$(BUILT_PRODUCTS_DIR)/DerivedSources/WebKit2/RemoteCDMFactoryProxyMessagesReplies.h
Modified: trunk/Source/WebKit/DerivedSources.make (268576 => 268577)
--- trunk/Source/WebKit/DerivedSources.make 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/DerivedSources.make 2020-10-16 09:48:57 UTC (rev 268577)
@@ -175,6 +175,7 @@
WebProcess/GPU/media/ios/RemoteMediaSessionHelper \
WebProcess/GPU/media/RemoteAudioDestinationProxy \
WebProcess/GPU/media/RemoteAudioSession \
+ WebProcess/GPU/media/RemoteAudioSourceProviderManager \
WebProcess/WebStorage/StorageAreaMap \
WebProcess/UserContent/WebUserContentController \
WebProcess/Inspector/WebInspectorInterruptDispatcher \
Added: trunk/Source/WebKit/GPUProcess/media/RemoteAudioSourceProviderProxy.cpp (0 => 268577)
--- trunk/Source/WebKit/GPUProcess/media/RemoteAudioSourceProviderProxy.cpp (rev 0)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteAudioSourceProviderProxy.cpp 2020-10-16 09:48:57 UTC (rev 268577)
@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2019-2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "RemoteAudioSourceProviderProxy.h"
+
+#if ENABLE(GPU_PROCESS) && ENABLE(WEB_AUDIO) && PLATFORM(COCOA)
+
+#include "RemoteAudioSourceProviderManagerMessages.h"
+#include <WebCore/AudioSourceProviderAVFObjC.h>
+
+namespace WebKit {
+
+Ref<RemoteAudioSourceProviderProxy> RemoteAudioSourceProviderProxy::create(WebCore::MediaPlayerIdentifier identifier, Ref<IPC::Connection>&& connection, AudioSourceProviderAVFObjC& localProvider)
+{
+ auto remoteProvider = adoptRef(*new RemoteAudioSourceProviderProxy(identifier, WTFMove(connection)));
+
+ localProvider.setRingBufferCreationCallback([remoteProvider](auto description, auto capacity) {
+ return remoteProvider->createRingBuffer(description, capacity);
+ });
+ localProvider.setAudioCallback([remoteProvider](auto startFrame, auto numberOfFrames) {
+ remoteProvider->newAudioSamples(startFrame, numberOfFrames);
+ });
+
+ return remoteProvider;
+}
+
+RemoteAudioSourceProviderProxy::RemoteAudioSourceProviderProxy(WebCore::MediaPlayerIdentifier identifier, Ref<IPC::Connection>&& connection)
+ : m_identifier(identifier)
+ , m_connection(WTFMove(connection))
+{
+}
+
+RemoteAudioSourceProviderProxy::~RemoteAudioSourceProviderProxy() = default;
+
+UniqueRef<CARingBuffer> RemoteAudioSourceProviderProxy::createRingBuffer(const CAAudioStreamDescription& description, size_t capacity)
+{
+ m_ringBufferDescription = description;
+ m_ringBufferCapacity = capacity;
+ auto ringBuffer = makeUniqueRef<CARingBuffer>(makeUniqueRef<SharedRingBufferStorage>(makeUniqueRef<SharedRingBufferStorage>(this)));
+ ringBuffer->allocate(description, capacity);
+ return ringBuffer;
+}
+
+void RemoteAudioSourceProviderProxy::newAudioSamples(uint64_t startFrame, uint64_t numberOfFrames)
+{
+ m_connection->send(Messages::RemoteAudioSourceProviderManager::AudioSamplesAvailable { m_identifier, startFrame, numberOfFrames }, 0);
+}
+
+void RemoteAudioSourceProviderProxy::storageChanged(SharedMemory* memory)
+{
+ SharedMemory::Handle handle;
+ if (memory)
+ memory->createHandle(handle, SharedMemory::Protection::ReadOnly);
+
+ // FIXME: Send the actual data size with IPCHandle.
+#if OS(DARWIN) || OS(WINDOWS)
+ uint64_t dataSize = handle.size();
+#else
+ uint64_t dataSize = 0;
+#endif
+ m_connection->send(Messages::RemoteAudioSourceProviderManager::AudioStorageChanged { m_identifier, SharedMemory::IPCHandle { WTFMove(handle), dataSize }, m_ringBufferDescription, m_ringBufferCapacity }, 0);
+}
+
+} // namespace WebKit
+
+#endif // ENABLE(GPU_PROCESS) && ENABLE(WEB_AUDIO) && PLATFORM(COCOA)
Added: trunk/Source/WebKit/GPUProcess/media/RemoteAudioSourceProviderProxy.h (0 => 268577)
--- trunk/Source/WebKit/GPUProcess/media/RemoteAudioSourceProviderProxy.h (rev 0)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteAudioSourceProviderProxy.h 2020-10-16 09:48:57 UTC (rev 268577)
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if ENABLE(GPU_PROCESS) && ENABLE(WEB_AUDIO) && PLATFORM(COCOA)
+
+#include "Connection.h"
+#include "SharedRingBufferStorage.h"
+#include <WebCore/AudioSourceProviderClient.h>
+#include <WebCore/CAAudioStreamDescription.h>
+#include <WebCore/MediaPlayerIdentifier.h>
+#include <wtf/ThreadSafeRefCounted.h>
+
+namespace WebCore {
+class AudioSourceProviderAVFObjC;
+class CARingBuffer;
+}
+
+namespace WebKit {
+
+class RemoteAudioSourceProviderProxy : public ThreadSafeRefCounted<RemoteAudioSourceProviderProxy>
+ , public SharedRingBufferStorage::Client
+ , public WebCore::AudioSourceProviderClient {
+public:
+ static Ref<RemoteAudioSourceProviderProxy> create(WebCore::MediaPlayerIdentifier, Ref<IPC::Connection>&&, WebCore::AudioSourceProviderAVFObjC&);
+ ~RemoteAudioSourceProviderProxy();
+
+ UniqueRef<WebCore::CARingBuffer> createRingBuffer(const WebCore::CAAudioStreamDescription&, size_t);
+ void newAudioSamples(uint64_t startFrame, uint64_t endFrame);
+
+private:
+ RemoteAudioSourceProviderProxy(WebCore::MediaPlayerIdentifier, Ref<IPC::Connection>&&);
+
+ // SharedRingBufferStorage::Client
+ void storageChanged(SharedMemory*) final;
+
+ // AudioSourceProviderClient
+ void setFormat(size_t numberOfChannels, float sampleRate) final { }
+
+ WebCore::MediaPlayerIdentifier m_identifier;
+ Ref<IPC::Connection> m_connection;
+
+ WebCore::CAAudioStreamDescription m_ringBufferDescription;
+ size_t m_ringBufferCapacity { 0 };
+};
+
+} // namespace WebKit
+
+#endif // ENABLE(GPU_PROCESS) && ENABLE(WEB_AUDIO) && PLATFORM(COCOA)
Modified: trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.cpp (268576 => 268577)
--- trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.cpp 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.cpp 2020-10-16 09:48:57 UTC (rev 268577)
@@ -32,6 +32,7 @@
#include "GPUConnectionToWebProcess.h"
#include "LayerHostingContext.h"
#include "MediaPlayerPrivateRemoteMessages.h"
+#include "RemoteAudioSourceProviderProxy.h"
#include "RemoteAudioTrackProxy.h"
#include "RemoteLegacyCDMFactoryProxy.h"
#include "RemoteLegacyCDMSessionProxy.h"
@@ -59,6 +60,10 @@
#include <WebCore/MediaPlaybackTargetMock.h>
#endif
+#if PLATFORM(COCOA)
+#include <WebCore/AudioSourceProviderAVFObjC.h>
+#endif
+
namespace WebKit {
using namespace WebCore;
@@ -82,6 +87,7 @@
{
if (m_performTaskAtMediaTimeCompletionHandler)
m_performTaskAtMediaTimeCompletionHandler(WTF::nullopt);
+ setShouldEnableAudioSourceProvider(false);
}
void RemoteMediaPlayerProxy::invalidate()
@@ -878,6 +884,28 @@
completionHandler(m_player->wouldTaintOrigin(originData.securityOrigin()));
}
+void RemoteMediaPlayerProxy::createAudioSourceProvider()
+{
+#if ENABLE(WEB_AUDIO) && PLATFORM(COCOA)
+ if (!m_player)
+ return;
+
+ auto* provider = m_player->audioSourceProvider();
+ if (!provider || !is<AudioSourceProviderAVFObjC>(provider))
+ return;
+
+ m_remoteAudioSourceProvider = RemoteAudioSourceProviderProxy::create(m_id, m_webProcessConnection.copyRef(), downcast<AudioSourceProviderAVFObjC>(*provider));
+#endif
+}
+
+void RemoteMediaPlayerProxy::setShouldEnableAudioSourceProvider(bool shouldEnable)
+{
+#if ENABLE(WEB_AUDIO) && PLATFORM(COCOA)
+ if (auto* provider = m_player->audioSourceProvider())
+ provider->setClient(shouldEnable ? m_remoteAudioSourceProvider.get() : nullptr);
+#endif
+}
+
} // namespace WebKit
#endif // ENABLE(GPU_PROCESS)
Modified: trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h (268576 => 268577)
--- trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.h 2020-10-16 09:48:57 UTC (rev 268577)
@@ -52,6 +52,10 @@
#include "RemoteCDMInstanceProxy.h"
#endif
+#if PLATFORM(COCOA)
+#include "SharedRingBufferStorage.h"
+#endif
+
namespace WTF {
class MachSendRight;
}
@@ -65,6 +69,7 @@
using LayerHostingContextID = uint32_t;
class LayerHostingContext;
+class RemoteAudioSourceProviderProxy;
class RemoteAudioTrackProxy;
class RemoteMediaPlayerManagerProxy;
class RemoteTextTrackProxy;
@@ -267,6 +272,9 @@
void sendCachedState();
void timerFired();
+ void createAudioSourceProvider();
+ void setShouldEnableAudioSourceProvider(bool);
+
#if !RELEASE_LOG_DISABLED
const Logger& mediaPlayerLogger() final { return m_logger; }
const void* mediaPlayerLogIdentifier() { return reinterpret_cast<const void*>(m_configuration.logIdentifier); }
@@ -303,6 +311,10 @@
RemoteLegacyCDMSessionIdentifier m_legacySession;
#endif
+#if ENABLE(WEB_AUDIO) && PLATFORM(COCOA)
+ RefPtr<RemoteAudioSourceProviderProxy> m_remoteAudioSourceProvider;
+#endif
+
#if !RELEASE_LOG_DISABLED
const Logger& m_logger;
#endif
Modified: trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.messages.in (268576 => 268577)
--- trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.messages.in 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/GPUProcess/media/RemoteMediaPlayerProxy.messages.in 2020-10-16 09:48:57 UTC (rev 268577)
@@ -116,6 +116,11 @@
ErrorLog() -> (String errorLog) Synchronous
AccessLog() -> (String accessLog) Synchronous
#endif
+
+#if ENABLE(WEB_AUDIO)
+ CreateAudioSourceProvider()
+ SetShouldEnableAudioSourceProvider(bool shouldEnable)
+#endif
}
#endif
Modified: trunk/Source/WebKit/SourcesCocoa.txt (268576 => 268577)
--- trunk/Source/WebKit/SourcesCocoa.txt 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/SourcesCocoa.txt 2020-10-16 09:48:57 UTC (rev 268577)
@@ -58,6 +58,7 @@
GPUProcess/EntryPoint/Cocoa/XPCService/GPUServiceEntryPoint.mm
GPUProcess/ios/GPUProcessIOS.mm
GPUProcess/mac/GPUProcessMac.mm
+GPUProcess/media/RemoteAudioSourceProviderProxy.cpp
GPUProcess/media/cocoa/RemoteMediaPlayerProxyCocoa.mm
GPUProcess/media/ios/RemoteMediaSessionHelperProxy.cpp
GPUProcess/webrtc/LibWebRTCCodecsProxy.mm
@@ -566,6 +567,7 @@
WebProcess/GPU/graphics/cocoa/ImageBufferShareableIOSurfaceBackend.cpp
WebProcess/GPU/media/RemoteAudioSourceProvider.cpp
+WebProcess/GPU/media/RemoteAudioSourceProviderManager.cpp
WebProcess/GPU/media/cocoa/MediaPlayerPrivateRemoteCocoa.mm
WebProcess/GPU/media/cocoa/VideoLayerRemoteCocoa.mm
WebProcess/GPU/media/ios/RemoteMediaSessionHelper.cpp
@@ -668,6 +670,7 @@
RemoteAudioSessionProxyMessageReceiver.cpp
RemoteAudioMediaStreamTrackRendererManagerMessageReceiver.cpp
RemoteAudioMediaStreamTrackRendererMessageReceiver.cpp
+RemoteAudioSourceProviderManagerMessageReceiver.cpp
RemoteMediaRecorderMessageReceiver.cpp
RemoteMediaRecorderManagerMessageReceiver.cpp
RemoteMediaSessionHelperMessageReceiver.cpp
Modified: trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj (268576 => 268577)
--- trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/WebKit.xcodeproj/project.pbxproj 2020-10-16 09:48:57 UTC (rev 268577)
@@ -3427,6 +3427,11 @@
41287D4D225C161F009A3E26 /* WebSocketTask.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = WebSocketTask.h; sourceTree = "<group>"; };
412FF91625373F9D001DF036 /* RemoteAudioSourceProvider.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RemoteAudioSourceProvider.cpp; sourceTree = "<group>"; };
412FF91725373F9D001DF036 /* RemoteAudioSourceProvider.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RemoteAudioSourceProvider.h; sourceTree = "<group>"; };
+ 412FF92325382BD3001DF036 /* RemoteAudioSourceProviderManager.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RemoteAudioSourceProviderManager.cpp; sourceTree = "<group>"; };
+ 412FF92425382BD5001DF036 /* RemoteAudioSourceProviderManager.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RemoteAudioSourceProviderManager.h; sourceTree = "<group>"; };
+ 412FF92525382BD5001DF036 /* RemoteAudioSourceProviderManager.messages.in */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = RemoteAudioSourceProviderManager.messages.in; sourceTree = "<group>"; };
+ 412FF92625382D85001DF036 /* RemoteAudioSourceProviderProxy.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RemoteAudioSourceProviderProxy.cpp; sourceTree = "<group>"; };
+ 412FF92725382D85001DF036 /* RemoteAudioSourceProviderProxy.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RemoteAudioSourceProviderProxy.h; sourceTree = "<group>"; };
4130759A1DE84FB00039EC69 /* NetworkRTCMonitor.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = NetworkRTCMonitor.cpp; sourceTree = "<group>"; };
4130759B1DE84FB00039EC69 /* NetworkRTCMonitor.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = NetworkRTCMonitor.h; sourceTree = "<group>"; };
4130759C1DE84FB00039EC69 /* NetworkRTCMonitor.messages.in */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = NetworkRTCMonitor.messages.in; sourceTree = "<group>"; };
@@ -4538,7 +4543,7 @@
9B1229D023FF2A5E008CA751 /* RemoteAudioDestinationIdentifier.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = RemoteAudioDestinationIdentifier.h; sourceTree = "<group>"; };
9B47908C25314D8300EC11AB /* MessageArgumentDescriptions.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MessageArgumentDescriptions.h; sourceTree = "<group>"; };
9B47908E253151CC00EC11AB /* JSIPCBinding.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = JSIPCBinding.h; sourceTree = "<group>"; };
- 9B4790902531563200EC11AB /* MessageArgumentDescriptions.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = MessageArgumentDescriptions.cpp; path = MessageArgumentDescriptions.cpp; sourceTree = "<group>"; };
+ 9B4790902531563200EC11AB /* MessageArgumentDescriptions.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = MessageArgumentDescriptions.cpp; sourceTree = "<group>"; };
9B5499AC2362A6F600DF8BA5 /* _WKTextManipulationConfiguration.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = _WKTextManipulationConfiguration.h; sourceTree = "<group>"; };
9B5499AE2362A7A700DF8BA5 /* _WKTextManipulationConfiguration.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = _WKTextManipulationConfiguration.mm; sourceTree = "<group>"; };
9B5499B02362A7EC00DF8BA5 /* _WKTextManipulationExclusionRule.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = _WKTextManipulationExclusionRule.h; sourceTree = "<group>"; };
@@ -5574,6 +5579,9 @@
CDBB49FB240D974A0017C292 /* RemoteAudioSessionIdentifier.h */,
412FF91625373F9D001DF036 /* RemoteAudioSourceProvider.cpp */,
412FF91725373F9D001DF036 /* RemoteAudioSourceProvider.h */,
+ 412FF92325382BD3001DF036 /* RemoteAudioSourceProviderManager.cpp */,
+ 412FF92425382BD5001DF036 /* RemoteAudioSourceProviderManager.h */,
+ 412FF92525382BD5001DF036 /* RemoteAudioSourceProviderManager.messages.in */,
CDAC20BF23FC2F750021DEE3 /* RemoteCDM.cpp */,
CDAC20BA23FC2F740021DEE3 /* RemoteCDM.h */,
CDAC20B823FC2F730021DEE3 /* RemoteCDMConfiguration.h */,
@@ -5629,6 +5637,8 @@
CDBB49F8240D8D530017C292 /* RemoteAudioSessionProxy.messages.in */,
CDBB49FA240D963D0017C292 /* RemoteAudioSessionProxyManager.cpp */,
CDBB49F9240D963C0017C292 /* RemoteAudioSessionProxyManager.h */,
+ 412FF92625382D85001DF036 /* RemoteAudioSourceProviderProxy.cpp */,
+ 412FF92725382D85001DF036 /* RemoteAudioSourceProviderProxy.h */,
071BC57E23CA5DB100680D7C /* RemoteAudioTrackProxy.cpp */,
071BC57F23CA5DB100680D7C /* RemoteAudioTrackProxy.h */,
CDAC207923FB1E210021DEE3 /* RemoteCDMFactoryProxy.cpp */,
Modified: trunk/Source/WebKit/WebProcess/GPU/GPUProcessConnection.cpp (268576 => 268577)
--- trunk/Source/WebKit/WebProcess/GPU/GPUProcessConnection.cpp 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/WebProcess/GPU/GPUProcessConnection.cpp 2020-10-16 09:48:57 UTC (rev 268577)
@@ -33,6 +33,7 @@
#include "LibWebRTCCodecs.h"
#include "LibWebRTCCodecsMessages.h"
#include "MediaPlayerPrivateRemoteMessages.h"
+#include "RemoteAudioSourceProviderManager.h"
#include "RemoteCDMFactory.h"
#include "RemoteCDMProxy.h"
#include "RemoteLegacyCDMFactory.h"
@@ -101,6 +102,15 @@
return *WebProcess::singleton().supplement<RemoteMediaPlayerManager>();
}
+#if PLATFORM(COCOA) && ENABLE(WEB_AUDIO)
+RemoteAudioSourceProviderManager& GPUProcessConnection::audioSourceProviderManager()
+{
+ if (!m_audioSourceProviderManager)
+ m_audioSourceProviderManager = RemoteAudioSourceProviderManager::create();
+ return *m_audioSourceProviderManager;
+}
+#endif
+
#if ENABLE(ENCRYPTED_MEDIA)
RemoteCDMFactory& GPUProcessConnection::cdmFactory()
{
Modified: trunk/Source/WebKit/WebProcess/GPU/GPUProcessConnection.h (268576 => 268577)
--- trunk/Source/WebKit/WebProcess/GPU/GPUProcessConnection.h 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/WebProcess/GPU/GPUProcessConnection.h 2020-10-16 09:48:57 UTC (rev 268577)
@@ -40,6 +40,7 @@
namespace WebKit {
+class RemoteAudioSourceProviderManager;
class RemoteCDMFactory;
class RemoteMediaPlayerManager;
class RemoteLegacyCDMFactory;
@@ -66,6 +67,10 @@
RemoteMediaPlayerManager& mediaPlayerManager();
+#if PLATFORM(COCOA) && ENABLE(WEB_AUDIO)
+ RemoteAudioSourceProviderManager& audioSourceProviderManager();
+#endif
+
#if ENABLE(ENCRYPTED_MEDIA)
RemoteCDMFactory& cdmFactory();
#endif
@@ -100,6 +105,9 @@
#if PLATFORM(COCOA) && ENABLE(MEDIA_STREAM)
std::unique_ptr<SampleBufferDisplayLayerManager> m_sampleBufferDisplayLayerManager;
#endif
+#if PLATFORM(COCOA) && ENABLE(WEB_AUDIO)
+ RefPtr<RemoteAudioSourceProviderManager> m_audioSourceProviderManager;
+#endif
#if ENABLE(VP9)
bool m_enableVP9Decoder { false };
bool m_enableVP9SWDecoder { false };
Modified: trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProvider.cpp (268576 => 268577)
--- trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProvider.cpp 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProvider.cpp 2020-10-16 09:48:57 UTC (rev 268577)
@@ -28,6 +28,9 @@
#if ENABLE(GPU_PROCESS) && ENABLE(WEB_AUDIO) && PLATFORM(COCOA)
+#include "RemoteAudioSourceProviderManager.h"
+#include "RemoteMediaPlayerProxyMessages.h"
+
namespace WebCore {
#if !RELEASE_LOG_DISABLED
extern WTFLogChannel LogMedia;
@@ -39,7 +42,16 @@
Ref<RemoteAudioSourceProvider> RemoteAudioSourceProvider::create(WebCore::MediaPlayerIdentifier identifier, WTF::LoggerHelper& helper)
{
- return adoptRef(*new RemoteAudioSourceProvider(identifier, helper));
+ auto provider = adoptRef(*new RemoteAudioSourceProvider(identifier, helper));
+
+ auto& gpuProcessConnection = WebProcess::singleton().ensureGPUProcessConnection();
+ gpuProcessConnection.audioSourceProviderManager().addProvider(provider.copyRef());
+
+#if ENABLE(WEB_AUDIO)
+ gpuProcessConnection.connection().send(Messages::RemoteMediaPlayerProxy::CreateAudioSourceProvider { }, provider->identifier());
+#endif
+
+ return provider;
}
RemoteAudioSourceProvider::RemoteAudioSourceProvider(MediaPlayerIdentifier identifier, WTF::LoggerHelper& helper)
@@ -49,6 +61,7 @@
, m_logIdentifier(helper.logIdentifier())
#endif
{
+ ASSERT(isMainThread());
UNUSED_PARAM(helper);
}
@@ -58,11 +71,13 @@
void RemoteAudioSourceProvider::close()
{
+ ASSERT(isMainThread());
+ WebProcess::singleton().ensureGPUProcessConnection().audioSourceProviderManager().removeProvider(m_identifier);
}
-void RemoteAudioSourceProvider::hasNewClient(AudioSourceProviderClient*)
+void RemoteAudioSourceProvider::hasNewClient(AudioSourceProviderClient* client)
{
- // FIXME: register/unregister as needed.
+ WebProcess::singleton().ensureGPUProcessConnection().connection().send(Messages::RemoteMediaPlayerProxy::SetShouldEnableAudioSourceProvider { !!client }, m_identifier);
}
void RemoteAudioSourceProvider::audioSamplesAvailable(const PlatformAudioData& data, const AudioStreamDescription& description, size_t size)
Added: trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProviderManager.cpp (0 => 268577)
--- trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProviderManager.cpp (rev 0)
+++ trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProviderManager.cpp 2020-10-16 09:48:57 UTC (rev 268577)
@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "RemoteAudioSourceProviderManager.h"
+
+#include "Logging.h"
+#include "RemoteAudioSourceProviderManagerMessages.h"
+#include "SharedRingBufferStorage.h"
+
+#if PLATFORM(COCOA) && ENABLE(GPU_PROCESS)
+
+namespace WebKit {
+using namespace PAL;
+using namespace WebCore;
+
+RemoteAudioSourceProviderManager::RemoteAudioSourceProviderManager()
+ : m_queue(WorkQueue::create("RemoteAudioSourceProviderManager", WorkQueue::Type::Serial, WorkQueue::QOS::UserInteractive))
+{
+}
+
+RemoteAudioSourceProviderManager::~RemoteAudioSourceProviderManager()
+{
+ setConnection(nullptr);
+}
+
+void RemoteAudioSourceProviderManager::setConnection(IPC::Connection* connection)
+{
+ if (m_connection == connection)
+ return;
+
+ if (m_connection)
+ m_connection->removeThreadMessageReceiver(Messages::RemoteAudioSourceProviderManager::messageReceiverName());
+
+ m_connection = WTFMove(connection);
+
+ if (m_connection)
+ m_connection->addThreadMessageReceiver(Messages::RemoteAudioSourceProviderManager::messageReceiverName(), this);
+}
+
+void RemoteAudioSourceProviderManager::addProvider(Ref<RemoteAudioSourceProvider>&& provider)
+{
+ ASSERT(WTF::isMainRunLoop());
+ setConnection(&WebProcess::singleton().ensureGPUProcessConnection().connection());
+
+ dispatchToThread([this, provider = WTFMove(provider)]() mutable {
+ auto identifier = provider->identifier();
+
+ ASSERT(!m_providers.contains(identifier));
+ m_providers.add(identifier, makeUnique<RemoteAudio>(WTFMove(provider)));
+ });
+}
+
+void RemoteAudioSourceProviderManager::removeProvider(MediaPlayerIdentifier identifier)
+{
+ ASSERT(WTF::isMainRunLoop());
+
+ dispatchToThread([this, identifier] {
+ ASSERT(m_providers.contains(identifier));
+ m_providers.remove(identifier);
+ });
+}
+
+void RemoteAudioSourceProviderManager::dispatchToThread(Function<void()>&& callback)
+{
+ m_queue->dispatch(WTFMove(callback));
+}
+
+void RemoteAudioSourceProviderManager::audioStorageChanged(MediaPlayerIdentifier identifier, const SharedMemory::IPCHandle& ipcHandle, const WebCore::CAAudioStreamDescription& description, uint64_t numberOfFrames)
+{
+ ASSERT(!WTF::isMainRunLoop());
+
+ auto iterator = m_providers.find(identifier);
+ if (iterator == m_providers.end()) {
+ RELEASE_LOG_ERROR(Media, "Unable to find provider %llu for storageChanged", identifier.toUInt64());
+ return;
+ }
+ iterator->value->setStorage(ipcHandle.handle, description, numberOfFrames);
+}
+
+void RemoteAudioSourceProviderManager::audioSamplesAvailable(MediaPlayerIdentifier identifier, uint64_t startFrame, uint64_t numberOfFrames)
+{
+ ASSERT(!WTF::isMainRunLoop());
+
+ auto iterator = m_providers.find(identifier);
+ if (iterator == m_providers.end()) {
+ RELEASE_LOG_ERROR(Media, "Unable to find provider %llu for audioSamplesAvailable", identifier.toUInt64());
+ return;
+ }
+ iterator->value->audioSamplesAvailable(startFrame, numberOfFrames);
+}
+
+RemoteAudioSourceProviderManager::RemoteAudio::RemoteAudio(Ref<RemoteAudioSourceProvider>&& provider)
+ : m_provider(WTFMove(provider))
+ , m_ringBuffer(makeUnique<CARingBuffer>(makeUniqueRef<SharedRingBufferStorage>(nullptr)))
+{
+}
+
+void RemoteAudioSourceProviderManager::RemoteAudio::setStorage(const SharedMemory::Handle& handle, const WebCore::CAAudioStreamDescription& description, uint64_t numberOfFrames)
+{
+ m_description = description;
+
+ RefPtr<SharedMemory> memory;
+ if (!handle.isNull()) {
+ memory = SharedMemory::map(handle, SharedMemory::Protection::ReadOnly);
+ RELEASE_LOG_ERROR_IF(!memory, Media, "Unable to create shared memory for audio provider %llu", m_provider->identifier().toUInt64());
+ }
+
+ auto& storage = static_cast<SharedRingBufferStorage&>(m_ringBuffer->storage());
+ if (!memory) {
+ m_ringBuffer->deallocate();
+ storage.setReadOnly(false);
+ storage.setStorage(nullptr);
+ return;
+ }
+
+ storage.setStorage(memory.releaseNonNull());
+ storage.setReadOnly(true);
+ m_ringBuffer->allocate(description, numberOfFrames);
+ m_buffer = makeUnique<WebAudioBufferList>(description, numberOfFrames);
+}
+
+void RemoteAudioSourceProviderManager::RemoteAudio::audioSamplesAvailable(uint64_t startFrame, uint64_t numberOfFrames)
+{
+ if (!m_buffer) {
+ RELEASE_LOG_ERROR(Media, "buffer for audio provider %llu is null", m_provider->identifier().toUInt64());
+ return;
+ }
+
+ if (!WebAudioBufferList::isSupportedDescription(m_description, numberOfFrames)) {
+ RELEASE_LOG_ERROR(Media, "Unable to support description with given number of frames for audio provider %llu", m_provider->identifier().toUInt64());
+ return;
+ }
+
+ auto endFrame = startFrame + numberOfFrames;
+ m_buffer->setSampleCount(numberOfFrames);
+
+ m_ringBuffer->setCurrentFrameBounds(startFrame, endFrame);
+ m_ringBuffer->fetch(m_buffer->list(), numberOfFrames, startFrame);
+
+ m_provider->audioSamplesAvailable(*m_buffer, m_description, numberOfFrames);
+}
+
+}
+
+#endif // PLATFORM(COCOA) && ENABLE(GPU_PROCESS)
Added: trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProviderManager.h (0 => 268577)
--- trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProviderManager.h (rev 0)
+++ trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProviderManager.h 2020-10-16 09:48:57 UTC (rev 268577)
@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if PLATFORM(COCOA) && ENABLE(GPU_PROCESS)
+
+#include <WebCore/CAAudioStreamDescription.h>
+#include <WebCore/CARingBuffer.h>
+#include <WebCore/MediaPlayerIdentifier.h>
+#include <WebCore/WebAudioBufferList.h>
+
+namespace WebKit {
+
+class RemoteAudioSourceProvider;
+
+class RemoteAudioSourceProviderManager : public IPC::Connection::ThreadMessageReceiverRefCounted {
+public:
+ static Ref<RemoteAudioSourceProviderManager> create() { return adoptRef(*new RemoteAudioSourceProviderManager()); }
+ ~RemoteAudioSourceProviderManager();
+
+ void addProvider(Ref<RemoteAudioSourceProvider>&&);
+ void removeProvider(WebCore::MediaPlayerIdentifier);
+
+ void didReceiveMessage(IPC::Connection&, IPC::Decoder&);
+
+private:
+ RemoteAudioSourceProviderManager();
+
+ // IPC::Connection::ThreadMessageReceiver
+ void dispatchToThread(Function<void()>&&) final;
+
+ // Messages
+ void audioStorageChanged(WebCore::MediaPlayerIdentifier, const SharedMemory::IPCHandle&, const WebCore::CAAudioStreamDescription&, uint64_t numberOfFrames);
+ void audioSamplesAvailable(WebCore::MediaPlayerIdentifier, uint64_t startFrame, uint64_t numberOfFrames);
+
+ void setConnection(IPC::Connection*);
+
+ class RemoteAudio {
+ WTF_MAKE_FAST_ALLOCATED;
+ public:
+ explicit RemoteAudio(Ref<RemoteAudioSourceProvider>&&);
+
+ void setStorage(const SharedMemory::Handle&, const WebCore::CAAudioStreamDescription&, uint64_t numberOfFrames);
+ void audioSamplesAvailable(uint64_t startFrame, uint64_t numberOfFrames);
+
+ private:
+ Ref<RemoteAudioSourceProvider> m_provider;
+ WebCore::CAAudioStreamDescription m_description;
+ std::unique_ptr<WebCore::CARingBuffer> m_ringBuffer;
+ std::unique_ptr<WebCore::WebAudioBufferList> m_buffer;
+ };
+
+ Ref<WorkQueue> m_queue;
+ RefPtr<IPC::Connection> m_connection;
+
+ // background thread member
+ HashMap<WebCore::MediaPlayerIdentifier, std::unique_ptr<RemoteAudio>> m_providers;
+};
+
+} // namespace WebKit
+
+#endif // PLATFORM(COCOA) && ENABLE(GPU_PROCESS)
Added: trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProviderManager.messages.in (0 => 268577)
--- trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProviderManager.messages.in (rev 0)
+++ trunk/Source/WebKit/WebProcess/GPU/media/RemoteAudioSourceProviderManager.messages.in 2020-10-16 09:48:57 UTC (rev 268577)
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2020 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#if PLATFORM(COCOA) && ENABLE(GPU_PROCESS)
+
+messages -> RemoteAudioSourceProviderManager NotRefCounted {
+ AudioStorageChanged(WebCore::MediaPlayerIdentifier id, WebKit::SharedMemory::IPCHandle storageHandle, WebCore::CAAudioStreamDescription description, uint64_t numberOfFrames)
+ AudioSamplesAvailable(WebCore::MediaPlayerIdentifier id, uint64_t startFrame, uint64_t numberOfFrames)
+}
+
+#endif
Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.cpp (268576 => 268577)
--- trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.cpp 2020-10-16 08:52:38 UTC (rev 268576)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.cpp 2020-10-16 09:48:57 UTC (rev 268577)
@@ -26,6 +26,7 @@
#include "config.h"
#include "RemoteCaptureSampleManager.h"
+#include "Logging.h"
#include "RemoteCaptureSampleManagerMessages.h"
#include "SharedRingBufferStorage.h"
#include <WebCore/WebAudioBufferList.h>
@@ -33,7 +34,6 @@
#if PLATFORM(COCOA) && ENABLE(MEDIA_STREAM)
namespace WebKit {
-using namespace PAL;
using namespace WebCore;
RemoteCaptureSampleManager::RemoteCaptureSampleManager()
