Diff
Modified: trunk/LayoutTests/ChangeLog (217561 => 217562)
--- trunk/LayoutTests/ChangeLog 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/LayoutTests/ChangeLog 2017-05-30 18:29:13 UTC (rev 217562)
@@ -1,3 +1,14 @@
+2017-05-30 Youenn Fablet <[email protected]>
+
+ RealtimeOutgoingVideoSource should not rotate muted frames
+ https://bugs.webkit.org/show_bug.cgi?id=172659
+
+ Reviewed by Eric Carlson.
+
+ * webrtc/routines.js:
+ * webrtc/video-rotation-expected.txt: Added.
+ * webrtc/video-rotation.html: Added.
+
2017-05-30 Ryan Haddad <[email protected]>
Mark js/intl-datetimeformat.html as failing.
Modified: trunk/LayoutTests/TestExpectations (217561 => 217562)
--- trunk/LayoutTests/TestExpectations 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/LayoutTests/TestExpectations 2017-05-30 18:29:13 UTC (rev 217562)
@@ -745,9 +745,11 @@
fast/mediastream/RTCPeerConnection-overloaded-operations-params.html [ Skip ]
fast/mediastream/RTCRtpSender-replaceTrack.html [ Skip ]
fast/mediastream/RTCPeerConnection-dtmf.html [ Failure ]
+webrtc/video-rotation.html [ Pass Failure ]
# libwebrtc backend doesn't generate a=bundle-only lines
fast/mediastream/RTCPeerConnection-inspect-offer-bundlePolicy-bundle-only.html [ Failure ]
+
# Need updating wptserve
imported/w3c/web-platform-tests/XMLHttpRequest/setrequestheader-content-type.htm [ Skip ]
imported/w3c/web-platform-tests/XMLHttpRequest/open-url-redirected-worker-origin.htm [ Skip ]
Modified: trunk/LayoutTests/webrtc/routines.js (217561 => 217562)
--- trunk/LayoutTests/webrtc/routines.js 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/LayoutTests/webrtc/routines.js 2017-05-30 18:29:13 UTC (rev 217562)
@@ -130,3 +130,18 @@
{
return new Promise((resolve) => setTimeout(resolve, duration));
}
+
+function waitForVideoSize(video, width, height, count)
+{
+ if (video.videoWidth === width && video.videoHeight === height)
+ return Promise.resolve("video has expected size");
+
+ if (count === undefined)
+ count = 0;
+ if (++count > 20)
+ return Promise.reject("waitForVideoSize timed out");
+
+ return waitFor(50).then(() => {
+ return waitForVideoSize(video, width, height, count);
+ });
+}
Added: trunk/LayoutTests/webrtc/video-rotation-expected.txt (0 => 217562)
--- trunk/LayoutTests/webrtc/video-rotation-expected.txt (rev 0)
+++ trunk/LayoutTests/webrtc/video-rotation-expected.txt 2017-05-30 18:29:13 UTC (rev 217562)
@@ -0,0 +1,7 @@
+
+
+PASS Setting video exchange
+PASS Track is enabled, video should not be black
+PASS Track is enabled and rotated, video should not be black and should change size
+PASS Track is disabled, video should be black
+
Added: trunk/LayoutTests/webrtc/video-rotation.html (0 => 217562)
--- trunk/LayoutTests/webrtc/video-rotation.html (rev 0)
+++ trunk/LayoutTests/webrtc/video-rotation.html 2017-05-30 18:29:13 UTC (rev 217562)
@@ -0,0 +1,99 @@
+<!doctype html>
+<html>
+ <head>
+ <meta charset="utf-8">
+ <title>Testing basic video exchange from offerer to receiver</title>
+ <script src=""
+ <script src=""
+ </head>
+ <body>
+ <video id="video" autoplay playsInline width="320" height="240"></video>
+ <canvas id="canvas1" width="320" height="240"></canvas>
+ <canvas id="canvas2" width="320" height="240"></canvas>
+ <canvas id="canvas3" width="320" height="240"></canvas>
+ <script src =""
+ <script>
+function isVideoBlack(id)
+{
+ var canvas = document.getElementById(id);
+ canvas.width = video.videoWidth;
+ canvas.height = video.videoHeight;
+ canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height);
+
+ imageData = canvas.getContext('2d').getImageData(0, 0, canvas.width, canvas.height);
+ data = ""
+ for (var cptr = 0; cptr < canvas.width * canvas.height; ++cptr) {
+ // Approximatively black pixels.
+ if (data[4 * cptr] > 30 || data[4 * cptr + 1] > 30 || data[4 * cptr + 2] > 30)
+ return false;
+ }
+ return true;
+}
+
+function pollVideoBlackCheck(expected, id, resolve)
+{
+ if (isVideoBlack(id) === expected) {
+ resolve();
+ return;
+ }
+
+ setTimeout(() => pollVideoBlackCheck(expected, id, resolve), 50);
+}
+
+function checkVideoBlack(expected, id)
+{
+ return new Promise((resolve, reject) => {
+ pollVideoBlackCheck(expected, id, resolve);
+ setTimeout(() => reject("checkVideoBlack timed out for '" + id + "', expected " + expected), 5000);
+ });
+}
+
+var track;
+var remoteTrack;
+promise_test((test) => {
+ if (window.testRunner)
+ testRunner.setUserMediaPermission(true);
+
+ return navigator.mediaDevices.getUserMedia({video: {width: 320, height: 240, facingMode: "environment"}}).then((localStream) => {
+ return new Promise((resolve, reject) => {
+ track = localStream.getVideoTracks()[0];
+
+ createConnections((firstConnection) => {
+ firstConnection.addTrack(track, localStream);
+ if (window.internals)
+ internals.applyRotationForOutgoingVideoSources(firstConnection);
+ }, (secondConnection) => {
+ secondConnection._ontrack_ = (trackEvent) => {
+ remoteTrack = trackEvent.track;
+ resolve(trackEvent.streams[0]);
+ };
+ });
+ setTimeout(() => reject("Test timed out"), 5000);
+ });
+ }).then((remoteStream) => {
+ video.srcObject = remoteStream;
+ return video.play();
+ });
+}, "Setting video exchange");
+
+promise_test((test) => {
+ return checkVideoBlack(false, "canvas1");
+}, "Track is enabled, video should not be black");
+
+promise_test((test) => {
+ if (window.internals)
+ window.internals.setCameraMediaStreamTrackOrientation(track, 90);
+
+ return checkVideoBlack(false, "canvas1").then(() => {
+ return waitForVideoSize(video, 240, 320);
+ });
+}, "Track is enabled and rotated, video should not be black and should change size");
+
+promise_test((test) => {
+ track.enabled = false;
+ return checkVideoBlack(true, "canvas2");
+}, "Track is disabled, video should be black");
+
+ </script>
+ </body>
+</html>
Modified: trunk/Source/WebCore/ChangeLog (217561 => 217562)
--- trunk/Source/WebCore/ChangeLog 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/ChangeLog 2017-05-30 18:29:13 UTC (rev 217562)
@@ -1,3 +1,36 @@
+2017-05-30 Youenn Fablet <[email protected]>
+
+ RealtimeOutgoingVideoSource should not rotate muted frames
+ https://bugs.webkit.org/show_bug.cgi?id=172659
+
+ Reviewed by Eric Carlson.
+
+ Test: webrtc/video-rotation.html
+
+ When sending black frames, it is unnecessary to rotate frames.
+ Instead, it is better to create frame with the expected width and height.
+ Making sure libwebrtc rotated images are not null, as only YUV rotation is supported.
+
+ Adding internals API to test part of that code path.
+ Full coverage should be added when mock source is producing YUV images.
+
+ * Modules/mediastream/PeerConnectionBackend.h:
+ (WebCore::PeerConnectionBackend::applyRotationForOutgoingVideoSources):
+ * Modules/mediastream/RTCPeerConnection.h:
+ * Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.cpp:
+ (WebCore::LibWebRTCPeerConnectionBackend::applyRotationForOutgoingVideoSources):
+ * Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.h:
+ * platform/mediastream/mac/RealtimeOutgoingVideoSource.cpp:
+ (WebCore::RealtimeOutgoingVideoSource::sendBlackFrames):
+ (WebCore::RealtimeOutgoingVideoSource::sendFrame):
+ * platform/mediastream/mac/RealtimeOutgoingVideoSource.h:
+ * testing/Internals.cpp:
+ (WebCore::Internals::Internals):
+ (WebCore::Internals::applyRotationForOutgoingVideoSources):
+ (WebCore::Internals::setCameraMediaStreamTrackOrientation):
+ * testing/Internals.h:
+ * testing/Internals.idl:
+
2017-05-30 Brent Fulgham <[email protected]>
Unreviewed typo correction.
Modified: trunk/Source/WebCore/Modules/mediastream/PeerConnectionBackend.h (217561 => 217562)
--- trunk/Source/WebCore/Modules/mediastream/PeerConnectionBackend.h 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/Modules/mediastream/PeerConnectionBackend.h 2017-05-30 18:29:13 UTC (rev 217562)
@@ -110,6 +110,8 @@
void disableICECandidateFiltering();
void enableICECandidateFiltering();
+ virtual void applyRotationForOutgoingVideoSources() { }
+
protected:
void fireICECandidateEvent(RefPtr<RTCIceCandidate>&&);
void doneGatheringCandidates();
Modified: trunk/Source/WebCore/Modules/mediastream/RTCPeerConnection.h (217561 => 217562)
--- trunk/Source/WebCore/Modules/mediastream/RTCPeerConnection.h 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/Modules/mediastream/RTCPeerConnection.h 2017-05-30 18:29:13 UTC (rev 217562)
@@ -155,6 +155,9 @@
void registerToController(RTCController&);
void unregisterFromController();
+ friend class Internals;
+ void applyRotationForOutgoingVideoSources() { m_backend->applyRotationForOutgoingVideoSources(); }
+
// EventTarget implementation.
void refEventTarget() final { ref(); }
void derefEventTarget() final { deref(); }
Modified: trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.cpp (217561 => 217562)
--- trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.cpp 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.cpp 2017-05-30 18:29:13 UTC (rev 217562)
@@ -375,6 +375,12 @@
return m_endpoint->getRTCRtpSenderParameters(sender);
}
+void LibWebRTCPeerConnectionBackend::applyRotationForOutgoingVideoSources()
+{
+ for (auto& source : m_videoSources)
+ source->setApplyRotation(true);
+}
+
} // namespace WebCore
#endif // USE(LIBWEBRTC)
Modified: trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.h (217561 => 217562)
--- trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.h 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.h 2017-05-30 18:29:13 UTC (rev 217562)
@@ -73,6 +73,7 @@
RTCRtpParameters getParameters(RTCRtpSender&) const final;
void emulatePlatformEvent(const String&) final { }
+ void applyRotationForOutgoingVideoSources() final;
friend LibWebRTCMediaEndpoint;
RTCPeerConnection& connection() { return m_peerConnection; }
Modified: trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.h (217561 => 217562)
--- trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.h 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.h 2017-05-30 18:29:13 UTC (rev 217562)
@@ -59,6 +59,7 @@
bool applySize(const IntSize&) final;
void orientationChanged(int orientation) final;
+ void monitorOrientation(OrientationNotifier&) final;
mutable RetainPtr<CGImageRef> m_previewImage;
mutable RetainPtr<PlatformLayer> m_previewLayer;
Modified: trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm (217561 => 217562)
--- trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm 2017-05-30 18:29:13 UTC (rev 217562)
@@ -143,8 +143,9 @@
auto pixelBuffer = pixelBufferFromCGImage(imageBuffer->copyImage()->nativeImage().get());
auto sampleBuffer = CMSampleBufferFromPixelBuffer(pixelBuffer.get());
-
- videoSampleAvailable(MediaSampleAVFObjC::create(sampleBuffer.get()));
+
+ // We use m_deviceOrientation to emulate sensor orientation
+ videoSampleAvailable(MediaSampleAVFObjC::create(sampleBuffer.get(), m_deviceOrientation));
}
bool MockRealtimeVideoSourceMac::applySize(const IntSize& newSize)
@@ -176,6 +177,12 @@
}
}
+void MockRealtimeVideoSourceMac::monitorOrientation(OrientationNotifier& notifier)
+{
+ notifier.addObserver(*this);
+ orientationChanged(notifier.orientation());
+}
+
} // namespace WebCore
#endif // ENABLE(MEDIA_STREAM)
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSource.cpp (217561 => 217562)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSource.cpp 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSource.cpp 2017-05-30 18:29:13 UTC (rev 217562)
@@ -138,7 +138,11 @@
void RealtimeOutgoingVideoSource::sendBlackFrames()
{
if (!m_blackFrame) {
- auto frame = m_bufferPool.CreateBuffer(m_width, m_height);
+ auto width = m_width;
+ auto height = m_height;
+ if (m_shouldApplyRotation && (m_currentRotation == webrtc::kVideoRotation_0 || m_currentRotation == webrtc::kVideoRotation_90))
+ std::swap(width, height);
+ auto frame = m_bufferPool.CreateBuffer(width, height);
frame->SetToBlack();
m_blackFrame = WTFMove(frame);
}
@@ -154,13 +158,7 @@
void RealtimeOutgoingVideoSource::sendFrame(rtc::scoped_refptr<webrtc::VideoFrameBuffer>&& buffer)
{
- // FIXME: We should make AVVideoCaptureSource handle the rotation whenever possible.
- if (m_shouldApplyRotation && m_currentRotation != webrtc::kVideoRotation_0) {
- // This implementation is inefficient, we should rotate on the CMSampleBuffer directly instead of doing this double allocation.
- buffer = buffer->NativeToI420Buffer();
- buffer = webrtc::I420Buffer::Rotate(*buffer, m_currentRotation);
- }
- webrtc::VideoFrame frame(buffer, 0, 0, m_currentRotation);
+ webrtc::VideoFrame frame(buffer, 0, 0, m_shouldApplyRotation ? webrtc::kVideoRotation_0 : m_currentRotation);
for (auto* sink : m_sinks)
sink->OnFrame(frame);
}
@@ -198,7 +196,15 @@
auto pixelFormatType = CVPixelBufferGetPixelFormatType(pixelBuffer);
if (pixelFormatType == kCVPixelFormatType_420YpCbCr8Planar || pixelFormatType == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
- sendFrame(new rtc::RefCountedObject<webrtc::CoreVideoFrameBuffer>(pixelBuffer));
+ rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = new rtc::RefCountedObject<webrtc::CoreVideoFrameBuffer>(pixelBuffer);
+ if (m_shouldApplyRotation && m_currentRotation != webrtc::kVideoRotation_0) {
+ // FIXME: We should make AVVideoCaptureSource handle the rotation whenever possible.
+ // This implementation is inefficient, we should rotate on the CMSampleBuffer directly instead of doing this double allocation.
+ auto rotatedBuffer = buffer->NativeToI420Buffer();
+ ASSERT(rotatedBuffer);
+ buffer = webrtc::I420Buffer::Rotate(*rotatedBuffer, m_currentRotation);
+ }
+ sendFrame(WTFMove(buffer));
return;
}
@@ -213,6 +219,8 @@
webrtc::ConvertToI420(webrtc::kBGRA, source, 0, 0, m_width, m_height, 0, webrtc::kVideoRotation_0, newBuffer);
}
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
+ if (m_shouldApplyRotation && m_currentRotation != webrtc::kVideoRotation_0)
+ newBuffer = webrtc::I420Buffer::Rotate(*newBuffer, m_currentRotation);
sendFrame(WTFMove(newBuffer));
}
Modified: trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSource.h (217561 => 217562)
--- trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSource.h 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingVideoSource.h 2017-05-30 18:29:13 UTC (rev 217562)
@@ -54,6 +54,8 @@
int AddRef() const final { ref(); return refCount(); }
int Release() const final { deref(); return refCount(); }
+ void setApplyRotation(bool shouldApplyRotation) { m_shouldApplyRotation = shouldApplyRotation; }
+
private:
RealtimeOutgoingVideoSource(Ref<RealtimeMediaSource>&&);
Modified: trunk/Source/WebCore/testing/Internals.cpp (217561 => 217562)
--- trunk/Source/WebCore/testing/Internals.cpp 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/testing/Internals.cpp 2017-05-30 18:29:13 UTC (rev 217562)
@@ -461,6 +461,9 @@
Internals::Internals(Document& document)
: ContextDestructionObserver(&document)
+#if ENABLE(MEDIA_STREAM)
+ , m_orientationNotifier(0)
+#endif
{
#if ENABLE(VIDEO_TRACK)
if (document.page())
@@ -1326,6 +1329,10 @@
object.stop();
}
+void Internals::applyRotationForOutgoingVideoSources(RTCPeerConnection& connection)
+{
+ connection.applyRotationForOutgoingVideoSources();
+}
#endif
#if ENABLE(MEDIA_STREAM)
@@ -4009,6 +4016,16 @@
}
#if ENABLE(MEDIA_STREAM)
+
+void Internals::setCameraMediaStreamTrackOrientation(MediaStreamTrack& track, int orientation)
+{
+ auto& source = track.source();
+ if (!source.isCaptureSource())
+ return;
+ m_orientationNotifier.orientationChanged(orientation);
+ source.monitorOrientation(m_orientationNotifier);
+}
+
void Internals::observeMediaStreamTrack(MediaStreamTrack& track)
{
m_track = &track;
Modified: trunk/Source/WebCore/testing/Internals.h (217561 => 217562)
--- trunk/Source/WebCore/testing/Internals.h 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/testing/Internals.h 2017-05-30 18:29:13 UTC (rev 217562)
@@ -30,6 +30,7 @@
#include "ContextDestructionObserver.h"
#include "ExceptionOr.h"
#include "JSDOMPromiseDeferred.h"
+#include "OrientationNotifer.h"
#include "PageConsoleClient.h"
#include "RealtimeMediaSource.h"
#include <runtime/Float32Array.h>
@@ -429,6 +430,7 @@
void setICECandidateFiltering(bool);
void setEnumeratingAllNetworkInterfacesEnabled(bool);
void stopPeerConnection(RTCPeerConnection&);
+ void applyRotationForOutgoingVideoSources(RTCPeerConnection&);
#endif
String getImageSourceURL(Element&);
@@ -570,6 +572,7 @@
void setPageVisibility(bool isVisible);
#if ENABLE(MEDIA_STREAM)
+ void setCameraMediaStreamTrackOrientation(MediaStreamTrack&, int orientation);
ExceptionOr<void> setMediaDeviceState(const String& id, const String& property, bool value);
unsigned long trackAudioSampleCount() const { return m_trackAudioSampleCount; }
unsigned long trackVideoSampleCount() const { return m_trackVideoSampleCount; }
@@ -591,6 +594,7 @@
void videoSampleAvailable(MediaSample&) final;
void audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t) final { m_trackAudioSampleCount++; }
+ OrientationNotifier m_orientationNotifier;
unsigned long m_trackVideoSampleCount { 0 };
unsigned long m_trackAudioSampleCount { 0 };
RefPtr<MediaStreamTrack> m_track;
Modified: trunk/Source/WebCore/testing/Internals.idl (217561 => 217562)
--- trunk/Source/WebCore/testing/Internals.idl 2017-05-30 17:47:29 UTC (rev 217561)
+++ trunk/Source/WebCore/testing/Internals.idl 2017-05-30 18:29:13 UTC (rev 217562)
@@ -533,6 +533,9 @@
void setPageVisibility(boolean isVisible);
+ [Conditional=WEB_RTC] void applyRotationForOutgoingVideoSources(RTCPeerConnection connection);
+
+ [Conditional=MEDIA_STREAM] void setCameraMediaStreamTrackOrientation(MediaStreamTrack track, short orientation);
[Conditional=MEDIA_STREAM] void observeMediaStreamTrack(MediaStreamTrack track);
[Conditional=MEDIA_STREAM] Promise<ImageData> grabNextMediaStreamTrackFrame();
[Conditional=MEDIA_STREAM] readonly attribute unsigned long trackAudioSampleCount;
