Diff
Modified: trunk/LayoutTests/ChangeLog (275829 => 275830)
--- trunk/LayoutTests/ChangeLog 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/LayoutTests/ChangeLog 2021-04-12 18:45:27 UTC (rev 275830)
@@ -1,3 +1,16 @@
+2021-04-12 Youenn Fablet <[email protected]>
+
+ Add support for RTCEncodedFrame metadata
+ https://bugs.webkit.org/show_bug.cgi?id=224430
+
+ Reviewed by Eric Carlson.
+
+ * http/wpt/webrtc/metadata-transform-expected.txt: Added.
+ * http/wpt/webrtc/metadata-transform-worker.js: Added.
+ (onrtctransform.process):
+ (onrtctransform):
+ * http/wpt/webrtc/metadata-transform.html: Added.
+
2021-04-12 Philippe Normand <[email protected]>
[GStreamer][WebRTC] An audio track should be muted when capture is interrupted by the OS.
Added: trunk/LayoutTests/http/wpt/webrtc/metadata-transform-expected.txt (0 => 275830)
--- trunk/LayoutTests/http/wpt/webrtc/metadata-transform-expected.txt (rev 0)
+++ trunk/LayoutTests/http/wpt/webrtc/metadata-transform-expected.txt 2021-04-12 18:45:27 UTC (rev 275830)
@@ -0,0 +1,5 @@
+
+
+PASS audio exchange with transform
+PASS video exchange with transform
+
Added: trunk/LayoutTests/http/wpt/webrtc/metadata-transform-worker.js (0 => 275830)
--- trunk/LayoutTests/http/wpt/webrtc/metadata-transform-worker.js (rev 0)
+++ trunk/LayoutTests/http/wpt/webrtc/metadata-transform-worker.js 2021-04-12 18:45:27 UTC (rev 275830)
@@ -0,0 +1,25 @@
+_onrtctransform_ = (event) => {
+ const transformer = event.transformer;
+ transformer.port._onmessage_ = (event) => transformer.port.postMessage(event.data);
+
+ transformer.reader = transformer.readable.getReader();
+ transformer.writer = transformer.writable.getWriter();
+
+ let isFirstFrame = true;
+ function process(transformer)
+ {
+ transformer.reader.read().then(chunk => {
+ if (chunk.done)
+ return;
+
+ if (isFirstFrame) {
+ isFirstFrame = false;
+ self.postMessage({ name: transformer.options.name, timestamp: chunk.value.timestamp, metadata: chunk.value.getMetadata() });
+ }
+ transformer.writer.write(chunk.value);
+ process(transformer);
+ });
+ }
+ process(transformer);
+};
+self.postMessage("registered");
Added: trunk/LayoutTests/http/wpt/webrtc/metadata-transform.html (0 => 275830)
--- trunk/LayoutTests/http/wpt/webrtc/metadata-transform.html (rev 0)
+++ trunk/LayoutTests/http/wpt/webrtc/metadata-transform.html 2021-04-12 18:45:27 UTC (rev 275830)
@@ -0,0 +1,85 @@
+<!doctype html>
+<html>
+ <head>
+ <meta charset="utf-8">
+ <script src=""
+ <script src=""
+ </head>
+ <body>
+ <video id="video1" autoplay></video>
+ <script src =""
+ <script>
+async function waitForMessage(worker, data)
+{
+ while (true) {
+ const received = await new Promise(resolve => worker._onmessage_ = (event) => resolve(event.data));
+ if (data ="" received)
+ return;
+ }
+}
+
+async function gatherMetadata(audio)
+{
+ worker = new Worker('metadata-transform-worker.js');
+ const data = "" new Promise(resolve => worker._onmessage_ = (event) => resolve(event.data));
+ assert_equals(data, "registered");
+
+ const localStream = await navigator.mediaDevices.getUserMedia({audio: audio, video: !audio});
+
+ let sender, receiver;
+ const senderTransform = new RTCRtpScriptTransform(worker, {name:'sender'});
+ const receiverTransform = new RTCRtpScriptTransform(worker, {name:'receiver'});
+
+ await new Promise((resolve, reject) => {
+ createConnections((firstConnection) => {
+ pc1 = firstConnection;
+ sender = firstConnection.addTrack(localStream.getTracks()[0], localStream);
+ sender.transform = senderTransform;
+ }, (secondConnection) => {
+ pc2 = secondConnection;
+ secondConnection._ontrack_ = (trackEvent) => {
+ receiver = trackEvent.receiver;
+ receiver.transform = receiverTransform;
+ resolve(trackEvent.streams[0]);
+ };
+ });
+ setTimeout(() => reject("Test timed out"), 5000);
+ });
+
+ return new Promise((resolve, reject) => {
+ let senderMetadata, senderTimestamp;
+ worker._onmessage_ = (event) => {
+ if (event.data.name === 'sender') {
+ senderMetadata = event.data.metadata;
+ senderTimestamp = event.data.timestamp;
+ } else if (event.data.name === 'receiver')
+ resolve([senderMetadata, senderTimestamp, event.data.metadata, event.data.timestamp]);
+ };
+ setTimeout(() => reject("Metadata test timed out"), 5000);
+ });
+}
+
+promise_test(async (test) => {
+ const [senderMetadata, senderTimestamp, receiverMetadata, receiverTimestamp] = await gatherMetadata(true);
+
+ assert_equals(senderTimestamp, receiverTimestamp, "timestamp");
+ assert_true(!!senderMetadata.synchronizationSource, "ssrc");
+ assert_equals(senderMetadata.synchronizationSource, receiverMetadata.synchronizationSource, "ssrc");
+ assert_array_equals(senderMetadata.contributingSources, receiverMetadata.contributingSources, "csrc");
+}, "audio exchange with transform");
+
+promise_test(async (test) => {
+ const [senderMetadata, senderTimestamp, receiverMetadata, receiverTimestamp] = await gatherMetadata(true);
+
+ assert_equals(senderTimestamp, receiverTimestamp, "timestamp");
+ assert_true(!!senderMetadata.synchronizationSource, "ssrc");
+ assert_equals(senderMetadata.synchronizationSource, receiverMetadata.synchronizationSource, "ssrc");
+ assert_array_equals(senderMetadata.contributingSources, receiverMetadata.contributingSources, "csrc");
+ assert_equals(senderMetadata.height, receiverMetadata.height, "height");
+ assert_equals(senderMetadata.width, receiverMetadata.width, "width");
+ assert_equals(senderMetadata.spatialIndex, receiverMetadata.spatialIndex, "spatialIndex");
+ assert_equals(senderMetadata.temporalIndex, receiverMetadata.temporalIndex, "temporalIndex");
+}, "video exchange with transform");
+ </script>
+ </body>
+</html>
Modified: trunk/Source/WebCore/ChangeLog (275829 => 275830)
--- trunk/Source/WebCore/ChangeLog 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/Source/WebCore/ChangeLog 2021-04-12 18:45:27 UTC (rev 275830)
@@ -1,3 +1,36 @@
+2021-04-12 Youenn Fablet <[email protected]>
+
+ Add support for RTCEncodedFrame metadata
+ https://bugs.webkit.org/show_bug.cgi?id=224430
+
+ Reviewed by Eric Carlson.
+
+ Expose metadata given from backend.
+ Audio sender side uses a regular TransformableFrameInterface instead of a TransformableAudioFrameInterface.
+ Make sure to not call GetHeader() in that case.
+
+ Test: http/wpt/webrtc/metadata-transform.html
+
+ * Modules/mediastream/RTCEncodedAudioFrame.cpp:
+ (WebCore::RTCEncodedAudioFrame::timestamp const):
+ (WebCore::RTCEncodedAudioFrame::getMetadata):
+ * Modules/mediastream/RTCEncodedAudioFrame.h:
+ * Modules/mediastream/RTCEncodedAudioFrame.idl:
+ * Modules/mediastream/RTCEncodedVideoFrame.cpp:
+ (WebCore::RTCEncodedVideoFrame::timestamp const):
+ (WebCore::RTCEncodedVideoFrame::getMetadata):
+ * Modules/mediastream/RTCEncodedVideoFrame.h:
+ * Modules/mediastream/RTCEncodedVideoFrame.idl:
+ * Modules/mediastream/RTCRtpTransformableFrame.h:
+ * Modules/mediastream/libwebrtc/LibWebRTCRtpTransformBackend.cpp:
+ (WebCore::LibWebRTCRtpTransformBackend::Transform):
+ * Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.cpp:
+ (WebCore::LibWebRTCRtpTransformableFrame::LibWebRTCRtpTransformableFrame):
+ (WebCore::LibWebRTCRtpTransformableFrame::timestamp const):
+ (WebCore::LibWebRTCRtpTransformableFrame::audioMetadata const):
+ (WebCore::LibWebRTCRtpTransformableFrame::videoMetadata const):
+ * Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.h:
+
2021-04-12 Sam Weinig <[email protected]>
Remove unused 'DefaultDefineOwnProperty' extended attribute support
Modified: trunk/Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.cpp (275829 => 275830)
--- trunk/Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.cpp 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.cpp 2021-04-12 18:45:27 UTC (rev 275830)
@@ -37,6 +37,18 @@
RTCEncodedAudioFrame::~RTCEncodedAudioFrame() = default;
+uint64_t RTCEncodedAudioFrame::timestamp() const
+{
+ return m_frame->timestamp();
+}
+
+const RTCEncodedAudioFrame::Metadata& RTCEncodedAudioFrame::getMetadata()
+{
+ if (!m_metadata)
+ m_metadata = m_frame->audioMetadata();
+ return *m_metadata;
+}
+
} // namespace WebCore
#endif // ENABLE(WEB_RTC)
Modified: trunk/Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.h (275829 => 275830)
--- trunk/Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.h 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.h 2021-04-12 18:45:27 UTC (rev 275830)
@@ -36,8 +36,15 @@
static Ref<RTCEncodedAudioFrame> create(Ref<RTCRtpTransformableFrame>&& frame) { return adoptRef(*new RTCEncodedAudioFrame(WTFMove(frame))); }
~RTCEncodedAudioFrame();
+ using Metadata = RTCEncodedAudioFrameMetadata;
+ const Metadata& getMetadata();
+
+ uint64_t timestamp() const;
+
private:
explicit RTCEncodedAudioFrame(Ref<RTCRtpTransformableFrame>&&);
+
+ mutable Optional<Metadata> m_metadata;
};
} // namespace WebCore
Modified: trunk/Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.idl (275829 => 275830)
--- trunk/Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.idl 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/Source/WebCore/Modules/mediastream/RTCEncodedAudioFrame.idl 2021-04-12 18:45:27 UTC (rev 275830)
@@ -25,11 +25,20 @@
[
Conditional=WEB_RTC,
+ JSGenerateToJSObject,
+]
+dictionary RTCEncodedAudioFrameMetadata {
+ unsigned long synchronizationSource;
+ sequence<unsigned long> contributingSources;
+};
+
+[
+ Conditional=WEB_RTC,
EnabledBySetting=WebRTCInsertableStreams,
Exposed=DedicatedWorker,
ImplementationLacksVTable,
] interface RTCEncodedAudioFrame {
- // readonly attribute unsigned long long timestamp;
+ readonly attribute unsigned long long timestamp;
attribute ArrayBuffer data;
- // RTCAudioFrameMetadata getMetadata();
+ RTCEncodedAudioFrameMetadata getMetadata();
};
Modified: trunk/Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.cpp (275829 => 275830)
--- trunk/Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.cpp 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.cpp 2021-04-12 18:45:27 UTC (rev 275830)
@@ -38,6 +38,18 @@
RTCEncodedVideoFrame::~RTCEncodedVideoFrame() = default;
+uint64_t RTCEncodedVideoFrame::timestamp() const
+{
+ return m_frame->timestamp();
+}
+
+const RTCEncodedVideoFrame::Metadata& RTCEncodedVideoFrame::getMetadata()
+{
+ if (!m_metadata)
+ m_metadata = m_frame->videoMetadata();
+ return *m_metadata;
+}
+
} // namespace WebCore
#endif // ENABLE(WEB_RTC)
Modified: trunk/Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.h (275829 => 275830)
--- trunk/Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.h 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.h 2021-04-12 18:45:27 UTC (rev 275830)
@@ -39,10 +39,16 @@
enum class Type { Empty, Key, Delta };
Type type() const { return m_type; }
+ using Metadata = RTCEncodedVideoFrameMetadata;
+ const Metadata& getMetadata();
+
+ uint64_t timestamp() const;
+
private:
explicit RTCEncodedVideoFrame(Ref<RTCRtpTransformableFrame>&&);
Type m_type;
+ mutable Optional<Metadata> m_metadata;
};
} // namespace WebCore
Modified: trunk/Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.idl (275829 => 275830)
--- trunk/Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.idl 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/Source/WebCore/Modules/mediastream/RTCEncodedVideoFrame.idl 2021-04-12 18:45:27 UTC (rev 275830)
@@ -27,12 +27,27 @@
[
Conditional=WEB_RTC,
+ JSGenerateToJSObject,
+]
+dictionary RTCEncodedVideoFrameMetadata {
+ long long frameId;
+ sequence<long long> dependencies;
+ unsigned short width;
+ unsigned short height;
+ long spatialIndex;
+ long temporalIndex;
+ unsigned long synchronizationSource;
+ sequence<unsigned long> contributingSources;
+};
+
+[
+ Conditional=WEB_RTC,
EnabledBySetting=WebRTCInsertableStreams,
Exposed=DedicatedWorker,
ImplementationLacksVTable,
] interface RTCEncodedVideoFrame {
readonly attribute RTCEncodedVideoFrameType type;
- // readonly attribute unsigned long long timestamp;
+ readonly attribute unsigned long long timestamp;
attribute ArrayBuffer data;
- // RTCVideoFrameMetadata getMetadata();
+ RTCEncodedVideoFrameMetadata getMetadata();
};
Modified: trunk/Source/WebCore/Modules/mediastream/RTCRtpTransformableFrame.h (275829 => 275830)
--- trunk/Source/WebCore/Modules/mediastream/RTCRtpTransformableFrame.h 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/Source/WebCore/Modules/mediastream/RTCRtpTransformableFrame.h 2021-04-12 18:45:27 UTC (rev 275830)
@@ -27,9 +27,26 @@
#if ENABLE(WEB_RTC)
#include <wtf/RefCounted.h>
+#include <wtf/Vector.h>
namespace WebCore {
+struct RTCEncodedAudioFrameMetadata {
+ uint32_t synchronizationSource;
+ Vector<uint32_t> contributingSources;
+};
+
+struct RTCEncodedVideoFrameMetadata {
+ Optional<int64_t> frameId;
+ Vector<int64_t> dependencies;
+ uint16_t width;
+ uint16_t height;
+ Optional<int32_t> spatialIndex;
+ Optional<int32_t> temporalIndex;
+ uint32_t synchronizationSource;
+ Vector<uint32_t> contributingSources;
+};
+
class RTCRtpTransformableFrame : public RefCounted<RTCRtpTransformableFrame> {
public:
virtual ~RTCRtpTransformableFrame() = default;
@@ -41,6 +58,10 @@
virtual Data data() const = 0;
virtual void setData(Data) = 0;
+ virtual uint64_t timestamp() const = 0;
+ virtual RTCEncodedAudioFrameMetadata audioMetadata() const = 0;
+ virtual RTCEncodedVideoFrameMetadata videoMetadata() const = 0;
+
virtual bool isKeyFrame() const = 0;
};
Modified: trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpTransformBackend.cpp (275829 => 275830)
--- trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpTransformBackend.cpp 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpTransformBackend.cpp 2021-04-12 18:45:27 UTC (rev 275830)
@@ -63,7 +63,7 @@
{
auto locker = holdLock(m_inputCallbackLock);
if (m_inputCallback) {
- m_inputCallback(LibWebRTCRtpTransformableFrame::create(WTFMove(rtcFrame)));
+ m_inputCallback(LibWebRTCRtpTransformableFrame::create(WTFMove(rtcFrame), m_mediaType == MediaType::Audio && m_side == Side::Sender));
return;
}
}
Modified: trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.cpp (275829 => 275830)
--- trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.cpp 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.cpp 2021-04-12 18:45:27 UTC (rev 275830)
@@ -37,8 +37,9 @@
namespace WebCore {
-LibWebRTCRtpTransformableFrame::LibWebRTCRtpTransformableFrame(std::unique_ptr<webrtc::TransformableFrameInterface>&& frame)
+LibWebRTCRtpTransformableFrame::LibWebRTCRtpTransformableFrame(std::unique_ptr<webrtc::TransformableFrameInterface>&& frame, bool isAudioSenderFrame)
: m_rtcFrame(WTFMove(frame))
+ , m_isAudioSenderFrame(isAudioSenderFrame)
{
}
@@ -72,6 +73,47 @@
return videoFrame && videoFrame->IsKeyFrame();
}
+uint64_t LibWebRTCRtpTransformableFrame::timestamp() const
+{
+ return m_rtcFrame ? m_rtcFrame->GetTimestamp() : 0;
+}
+
+RTCEncodedAudioFrameMetadata LibWebRTCRtpTransformableFrame::audioMetadata() const
+{
+ if (!m_rtcFrame)
+ return { };
+
+ Vector<uint32_t> cssrcs;
+ if (!m_isAudioSenderFrame) {
+ auto* audioFrame = static_cast<webrtc::TransformableAudioFrameInterface*>(m_rtcFrame.get());
+ auto& header = audioFrame->GetHeader();
+ if (header.numCSRCs) {
+ cssrcs.reserveInitialCapacity(header.numCSRCs);
+ for (size_t cptr = 0; cptr < header.numCSRCs; ++cptr)
+ cssrcs.uncheckedAppend(header.arrOfCSRCs[cptr]);
+ }
+ }
+ return { m_rtcFrame->GetSsrc(), WTFMove(cssrcs) };
+}
+
+RTCEncodedVideoFrameMetadata LibWebRTCRtpTransformableFrame::videoMetadata() const
+{
+ if (!m_rtcFrame)
+ return { };
+ auto* videoFrame = static_cast<webrtc::TransformableVideoFrameInterface*>(m_rtcFrame.get());
+ auto& metadata = videoFrame->GetMetadata();
+
+ Optional<int64_t> frameId;
+ if (metadata.GetFrameId())
+ frameId = *metadata.GetFrameId();
+
+ Vector<int64_t> dependencies;
+ for (auto value : metadata.GetFrameDependencies())
+ dependencies.append(value);
+
+ return { frameId, WTFMove(dependencies), metadata.GetWidth(), metadata.GetHeight(), metadata.GetSpatialIndex(), metadata.GetTemporalIndex(), m_rtcFrame->GetSsrc(), { } };
+}
+
} // namespace WebCore
#endif // ENABLE(WEB_RTC) && USE(LIBWEBRTC)
Modified: trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.h (275829 => 275830)
--- trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.h 2021-04-12 17:37:11 UTC (rev 275829)
+++ trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCRtpTransformableFrame.h 2021-04-12 18:45:27 UTC (rev 275830)
@@ -38,20 +38,24 @@
class LibWebRTCRtpTransformableFrame final : public RTCRtpTransformableFrame {
WTF_MAKE_FAST_ALLOCATED;
public:
- static Ref<LibWebRTCRtpTransformableFrame> create(std::unique_ptr<webrtc::TransformableFrameInterface>&& frame) { return adoptRef(*new LibWebRTCRtpTransformableFrame(WTFMove(frame))); }
+ static Ref<LibWebRTCRtpTransformableFrame> create(std::unique_ptr<webrtc::TransformableFrameInterface>&& frame, bool isAudioSenderFrame) { return adoptRef(*new LibWebRTCRtpTransformableFrame(WTFMove(frame), isAudioSenderFrame)); }
~LibWebRTCRtpTransformableFrame();
std::unique_ptr<webrtc::TransformableFrameInterface> takeRTCFrame();
private:
- explicit LibWebRTCRtpTransformableFrame(std::unique_ptr<webrtc::TransformableFrameInterface>&&);
+ LibWebRTCRtpTransformableFrame(std::unique_ptr<webrtc::TransformableFrameInterface>&&, bool isAudioSenderFrame);
// RTCRtpTransformableFrame
Data data() const final;
void setData(Data) final;
bool isKeyFrame() const final;
+ uint64_t timestamp() const final;
+ RTCEncodedAudioFrameMetadata audioMetadata() const final;
+ RTCEncodedVideoFrameMetadata videoMetadata() const final;
std::unique_ptr<webrtc::TransformableFrameInterface> m_rtcFrame;
+ bool m_isAudioSenderFrame;
};
} // namespace WebCore
