[webkit-changes] [295524] trunk/Source/WebKit

Tue, 14 Jun 2022 09:22:38 -0700

Title: [295524] trunk/Source/WebKit
Revision
295524
Author
commit-qu...@webkit.org
Date
2022-06-14 09:21:47 -0700 (Tue, 14 Jun 2022)

Log Message

UserMediaCaptureManagerProxy should allow reusing any camera source of the same page
https://bugs.webkit.org/show_bug.cgi?id=241548
rdar://problem/95091899

Patch by Youenn Fablet <youe...@gmail.com> on 2022-06-14
Reviewed by Eric Carlson.

We are trying to reuse an existing camera source but we were only storing one camera source.
This does not work well in case the camera source gets cloned and then ended.
What we need to do is store all sources and if one is reusable, use it.

We use a WeakHashSet to store all camera sources (through getUserMedia or clone)
and iterate on them to either reuse and if not possible stop them.

This also ensures that a page does not have two live AVVideoCaptureSource sources.
If we had that, one source would interrupt the other and vice versa.

Fix a bug in which a cloned track would not have its PageIdentifier.

* Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp:
(WebKit::UserMediaCaptureManagerProxy::createCameraSource):
(WebKit::UserMediaCaptureManagerProxy::clone):
* Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.h:
* Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.messages.in:
* Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSourceProxy.cpp:
(WebKit::RemoteRealtimeMediaSourceProxy::createRemoteCloneSource):
* Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSourceProxy.h:
* Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.cpp:
(WebKit::RemoteRealtimeVideoSource::clone):

Canonical link: https://commits.webkit.org/251529@main

Modified Paths

Diff

Modified: trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp (295523 => 295524)


--- trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp	2022-06-14 16:19:43 UTC (rev 295523)
+++ trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.cpp	2022-06-14 16:21:47 UTC (rev 295524)
@@ -302,17 +302,17 @@
 CaptureSourceOrError UserMediaCaptureManagerProxy::createCameraSource(const CaptureDevice& device, String&& hashSalt, const MediaConstraints* mediaConstraints, PageIdentifier pageIdentifier)
 {
     auto& perPageSources = m_pageSources.ensure(pageIdentifier, [] { return PageSources { }; }).iterator->value;
-    auto* cameraSource = perPageSources.cameraSource.get();
-    if (cameraSource) {
+    for (auto& cameraSource : perPageSources.cameraSources) {
         // FIXME: Optimize multiple concurrent cameras.
-        if (cameraSource->persistentID() == device.persistentId() && !cameraSource->isEnded()) {
+        if (cameraSource.persistentID() == device.persistentId() && !cameraSource.isEnded()) {
             // We can reuse the source, let's do it.
-            auto source = cameraSource->clone();
+            auto source = cameraSource.clone();
             if (mediaConstraints) {
                 auto error = source->applyConstraints(*mediaConstraints);
                 if (error)
                     return WTFMove(error->message);
             }
+            perPageSources.cameraSources.add(source.get());
             return source;
         }
     }
@@ -321,15 +321,17 @@
     if (!sourceOrError)
         return sourceOrError;
 
-    if (!canCaptureFromMultipleCameras() && cameraSource) {
-        RELEASE_LOG_ERROR(WebRTC, "Ending camera source as new source is using a different device.");
-        // FIXME: We should probably fail the capture in a way that shows a specific console log message.
-        cameraSource->endImmediatly();
+    if (!canCaptureFromMultipleCameras()) {
+        perPageSources.cameraSources.forEach([](auto& source) {
+            RELEASE_LOG_ERROR(WebRTC, "Ending camera source as new source is using a different device.");
+            // FIXME: We should probably fail the capture in a way that shows a specific console log message.
+            source.endImmediatly();
+        });
     }
 
     auto source = sourceOrError.source();
     source->monitorOrientation(m_orientationNotifier);
-    perPageSources.cameraSource = WeakPtr { source.get() };
+    perPageSources.cameraSources.add(source.get());
     return source;
 }
 
@@ -443,12 +445,17 @@
         m_connectionProxy->connection().send(Messages::UserMediaCaptureManager::ApplyConstraintsFailed(id, result->badConstraint, result->message), 0);
 }
 
-void UserMediaCaptureManagerProxy::clone(RealtimeMediaSourceIdentifier clonedID, RealtimeMediaSourceIdentifier newSourceID)
+void UserMediaCaptureManagerProxy::clone(RealtimeMediaSourceIdentifier clonedID, RealtimeMediaSourceIdentifier newSourceID, WebCore::PageIdentifier pageIdentifier)
 {
     MESSAGE_CHECK(m_proxies.contains(clonedID));
     MESSAGE_CHECK(!m_proxies.contains(newSourceID));
-    if (auto* proxy = m_proxies.get(clonedID))
-        m_proxies.add(newSourceID, makeUnique<SourceProxy>(newSourceID, m_connectionProxy->connection(), ProcessIdentity { m_connectionProxy->resourceOwner() }, proxy->source().clone(), m_connectionProxy->remoteVideoFrameObjectHeap()));
+    if (auto* proxy = m_proxies.get(clonedID)) {
+        auto sourceClone = proxy->source().clone();
+        if (sourceClone->deviceType() == WebCore::CaptureDevice::DeviceType::Camera)
+            m_pageSources.ensure(pageIdentifier, [] { return PageSources { }; }).iterator->value.cameraSources.add(sourceClone.get());
+
+        m_proxies.add(newSourceID, makeUnique<SourceProxy>(newSourceID, m_connectionProxy->connection(), ProcessIdentity { m_connectionProxy->resourceOwner() }, WTFMove(sourceClone), m_connectionProxy->remoteVideoFrameObjectHeap()));
+    }
 }
 
 void UserMediaCaptureManagerProxy::endProducingData(RealtimeMediaSourceIdentifier sourceID)

Modified: trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.h (295523 => 295524)


--- trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.h	2022-06-14 16:19:43 UTC (rev 295523)
+++ trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.h	2022-06-14 16:21:47 UTC (rev 295524)
@@ -90,7 +90,7 @@
     void removeSource(WebCore::RealtimeMediaSourceIdentifier);
     void capabilities(WebCore::RealtimeMediaSourceIdentifier, CompletionHandler<void(WebCore::RealtimeMediaSourceCapabilities&&)>&&);
     void applyConstraints(WebCore::RealtimeMediaSourceIdentifier, const WebCore::MediaConstraints&);
-    void clone(WebCore::RealtimeMediaSourceIdentifier clonedID, WebCore::RealtimeMediaSourceIdentifier cloneID);
+    void clone(WebCore::RealtimeMediaSourceIdentifier clonedID, WebCore::RealtimeMediaSourceIdentifier cloneID, WebCore::PageIdentifier);
     void endProducingData(WebCore::RealtimeMediaSourceIdentifier);
     void setShouldApplyRotation(WebCore::RealtimeMediaSourceIdentifier, bool shouldApplyRotation);
 
@@ -105,7 +105,7 @@
 
     struct PageSources {
         WeakPtr<WebCore::RealtimeMediaSource> microphoneSource;
-        WeakPtr<WebCore::RealtimeMediaSource> cameraSource;
+        WeakHashSet<WebCore::RealtimeMediaSource> cameraSources;
     };
     HashMap<WebCore::PageIdentifier, PageSources> m_pageSources;
 };

Modified: trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.messages.in (295523 => 295524)


--- trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.messages.in	2022-06-14 16:19:43 UTC (rev 295523)
+++ trunk/Source/WebKit/UIProcess/Cocoa/UserMediaCaptureManagerProxy.messages.in	2022-06-14 16:21:47 UTC (rev 295524)
@@ -29,7 +29,7 @@
     StopProducingData(WebCore::RealtimeMediaSourceIdentifier id)
     RemoveSource(WebCore::RealtimeMediaSourceIdentifier id)
     ApplyConstraints(WebCore::RealtimeMediaSourceIdentifier id, struct WebCore::MediaConstraints constraints)
-    Clone(WebCore::RealtimeMediaSourceIdentifier clonedID, WebCore::RealtimeMediaSourceIdentifier cloneID)
+    Clone(WebCore::RealtimeMediaSourceIdentifier clonedID, WebCore::RealtimeMediaSourceIdentifier cloneID, WebCore::PageIdentifier pageIdentifier)
     EndProducingData(WebCore::RealtimeMediaSourceIdentifier sourceID)
     SetShouldApplyRotation(WebCore::RealtimeMediaSourceIdentifier sourceID, bool shouldApplyRotation)
 }

Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSource.cpp (295523 => 295524)


--- trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSource.cpp	2022-06-14 16:19:43 UTC (rev 295523)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSource.cpp	2022-06-14 16:21:47 UTC (rev 295524)
@@ -60,8 +60,8 @@
 {
 }
 
-RemoteRealtimeMediaSource::RemoteRealtimeMediaSource(RemoteRealtimeMediaSourceProxy&& proxy, AtomString&& name, String&& hashSalt, UserMediaCaptureManager& manager)
-    : RealtimeMediaSource(toSourceType(proxy.device().type()), WTFMove(name), String { proxy.device().persistentId() }, WTFMove(hashSalt), proxy.pageIdentifier())
+RemoteRealtimeMediaSource::RemoteRealtimeMediaSource(RemoteRealtimeMediaSourceProxy&& proxy, AtomString&& name, String&& hashSalt, UserMediaCaptureManager& manager, PageIdentifier pageIdentifier)
+    : RealtimeMediaSource(toSourceType(proxy.device().type()), WTFMove(name), String { proxy.device().persistentId() }, WTFMove(hashSalt), pageIdentifier)
     , m_proxy(WTFMove(proxy))
     , m_manager(manager)
 {

Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSource.h (295523 => 295524)


--- trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSource.h	2022-06-14 16:19:43 UTC (rev 295523)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSource.h	2022-06-14 16:21:47 UTC (rev 295524)
@@ -42,7 +42,7 @@
 {
 public:
     RemoteRealtimeMediaSource(WebCore::RealtimeMediaSourceIdentifier, const WebCore::CaptureDevice&, const WebCore::MediaConstraints*, AtomString&& name, String&& hashSalt, UserMediaCaptureManager&, bool shouldCaptureInGPUProcess, WebCore::PageIdentifier);
-    RemoteRealtimeMediaSource(RemoteRealtimeMediaSourceProxy&&, AtomString&& name, String&& hashSalt, UserMediaCaptureManager&);
+    RemoteRealtimeMediaSource(RemoteRealtimeMediaSourceProxy&&, AtomString&& name, String&& hashSalt, UserMediaCaptureManager&, WebCore::PageIdentifier);
 
     WebCore::RealtimeMediaSourceIdentifier identifier() const { return m_proxy.identifier(); }
     IPC::Connection& connection() { return m_proxy.connection(); }

Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSourceProxy.cpp (295523 => 295524)


--- trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSourceProxy.cpp	2022-06-14 16:19:43 UTC (rev 295523)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSourceProxy.cpp	2022-06-14 16:21:47 UTC (rev 295524)
@@ -100,9 +100,9 @@
     return { RealtimeMediaSourceIdentifier::generate(), m_device, m_shouldCaptureInGPUProcess, &m_constraints };
 }
 
-void RemoteRealtimeMediaSourceProxy::createRemoteCloneSource(WebCore::RealtimeMediaSourceIdentifier cloneIdentifier)
+void RemoteRealtimeMediaSourceProxy::createRemoteCloneSource(WebCore::RealtimeMediaSourceIdentifier cloneIdentifier, WebCore::PageIdentifier pageIdentifier)
 {
-    m_connection->send(Messages::UserMediaCaptureManagerProxy::Clone { m_identifier, cloneIdentifier }, 0);
+    m_connection->send(Messages::UserMediaCaptureManagerProxy::Clone { m_identifier, cloneIdentifier, pageIdentifier }, 0);
 }
 
 void RemoteRealtimeMediaSourceProxy::applyConstraints(const MediaConstraints& constraints, RealtimeMediaSource::ApplyConstraintsHandler&& completionHandler)

Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSourceProxy.h (295523 => 295524)


--- trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSourceProxy.h	2022-06-14 16:19:43 UTC (rev 295523)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeMediaSourceProxy.h	2022-06-14 16:21:47 UTC (rev 295524)
@@ -52,7 +52,6 @@
 
     IPC::Connection& connection() { return m_connection.get(); }
     WebCore::RealtimeMediaSourceIdentifier identifier() const { return m_identifier; }
-    WebCore::PageIdentifier pageIdentifier() const { return m_pageIdentifier; }
     WebCore::CaptureDevice::DeviceType deviceType() const { return m_device.type(); }
     const WebCore::CaptureDevice& device() const { return m_device; }
     bool shouldCaptureInGPUProcess() const { return m_shouldCaptureInGPUProcess; }
@@ -61,7 +60,7 @@
     void createRemoteMediaSource(const String&, WebCore::PageIdentifier, CreateCallback&&, bool shouldUseRemoteFrame = false);
 
     RemoteRealtimeMediaSourceProxy clone();
-    void createRemoteCloneSource(WebCore::RealtimeMediaSourceIdentifier);
+    void createRemoteCloneSource(WebCore::RealtimeMediaSourceIdentifier, WebCore::PageIdentifier);
 
     void applyConstraintsSucceeded();
     void applyConstraintsFailed(String&& failedConstraint, String&& errorMessage);
@@ -91,7 +90,6 @@
     Ref<IPC::Connection> m_connection;
     WebCore::CaptureDevice m_device;
     bool m_shouldCaptureInGPUProcess { false };
-    WebCore::PageIdentifier m_pageIdentifier;
 
     WebCore::MediaConstraints m_constraints;
     Deque<WebCore::RealtimeMediaSource::ApplyConstraintsHandler> m_pendingApplyConstraintsCallbacks;

Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.cpp (295523 => 295524)


--- trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.cpp	2022-06-14 16:19:43 UTC (rev 295523)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.cpp	2022-06-14 16:21:47 UTC (rev 295524)
@@ -45,11 +45,13 @@
 RemoteRealtimeVideoSource::RemoteRealtimeVideoSource(RealtimeMediaSourceIdentifier identifier, const CaptureDevice& device, const MediaConstraints* constraints, String&& hashSalt, UserMediaCaptureManager& manager, bool shouldCaptureInGPUProcess, PageIdentifier pageIdentifier)
     : RemoteRealtimeMediaSource(identifier, device, constraints, AtomString { device.label() }, WTFMove(hashSalt), manager, shouldCaptureInGPUProcess, pageIdentifier)
 {
+    ASSERT(this->pageIdentifier());
 }
 
-RemoteRealtimeVideoSource::RemoteRealtimeVideoSource(RemoteRealtimeMediaSourceProxy&& proxy, AtomString&& name, String&& hashSalt, UserMediaCaptureManager& manager)
-    : RemoteRealtimeMediaSource(WTFMove(proxy), WTFMove(name), WTFMove(hashSalt), manager)
+RemoteRealtimeVideoSource::RemoteRealtimeVideoSource(RemoteRealtimeMediaSourceProxy&& proxy, AtomString&& name, String&& hashSalt, UserMediaCaptureManager& manager, PageIdentifier pageIdentifier)
+    : RemoteRealtimeMediaSource(WTFMove(proxy), WTFMove(name), WTFMove(hashSalt), manager, pageIdentifier)
 {
+    ASSERT(this->pageIdentifier());
 }
 
 RemoteRealtimeVideoSource::~RemoteRealtimeVideoSource()
@@ -73,7 +75,7 @@
     if (isEnded() || proxy().isEnded())
         return *this;
 
-    auto source = adoptRef(*new RemoteRealtimeVideoSource(proxy().clone(), AtomString { name() }, deviceIDHashSalt(), manager()));
+    auto source = adoptRef(*new RemoteRealtimeVideoSource(proxy().clone(), AtomString { name() }, deviceIDHashSalt(), manager(), pageIdentifier()));
 
     source->setSettings(RealtimeMediaSourceSettings { settings() });
     source->setCapabilities(RealtimeMediaSourceCapabilities { capabilities() });
@@ -80,7 +82,7 @@
 
     manager().addSource(source.copyRef());
     manager().remoteCaptureSampleManager().addSource(source.copyRef());
-    proxy().createRemoteCloneSource(source->identifier());
+    proxy().createRemoteCloneSource(source->identifier(), pageIdentifier());
 
     return source;
 }

Modified: trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.h (295523 => 295524)


--- trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.h	2022-06-14 16:19:43 UTC (rev 295523)
+++ trunk/Source/WebKit/WebProcess/cocoa/RemoteRealtimeVideoSource.h	2022-06-14 16:21:47 UTC (rev 295524)
@@ -50,7 +50,7 @@
 
 private:
     RemoteRealtimeVideoSource(WebCore::RealtimeMediaSourceIdentifier, const WebCore::CaptureDevice&, const WebCore::MediaConstraints*, String&& hashSalt, UserMediaCaptureManager&, bool shouldCaptureInGPUProcess, WebCore::PageIdentifier);
-    RemoteRealtimeVideoSource(RemoteRealtimeMediaSourceProxy&&, AtomString&& name, String&& hashSalt, UserMediaCaptureManager&);
+    RemoteRealtimeVideoSource(RemoteRealtimeMediaSourceProxy&&, AtomString&& name, String&& hashSalt, UserMediaCaptureManager&, WebCore::PageIdentifier);
 
     Ref<RealtimeMediaSource> clone() final;
     void endProducingData() final;

_______________________________________________
webkit-changes mailing list
webkit-changes@lists.webkit.org
https://lists.webkit.org/mailman/listinfo/webkit-changes

Reply via email to