- Revision
- 283585
- Author
- [email protected]
- Date
- 2021-10-05 16:43:27 -0700 (Tue, 05 Oct 2021)
Log Message
createImageBitmap using a HLS video as source always return a black image.
https://bugs.webkit.org/show_bug.cgi?id=231225
rdar://83884031
Source/WebCore:
When playing HLS content, [AVURLAsset tracks] return an empty array.
We need to instead retrieve it from the AVPlayerItem object. The method
paintWithVideoOutput would have bailed out early as a consequence.
So we refactor the code a little to retrieve the tracks where they can be found
Reviewed by Eric Carlson.
Test: http/tests/media/video-hls-copy-into-canvas.html
* platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h:
* platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm:
(WebCore::MediaPlayerPrivateAVFoundationObjC::createAVPlayerItem):
(WebCore::MediaPlayerPrivateAVFoundationObjC::tracksChanged):
(WebCore::MediaPlayerPrivateAVFoundationObjC::updateRotationSession):
(WebCore::MediaPlayerPrivateAVFoundationObjC::audioSourceProvider):
(WebCore::MediaPlayerPrivateAVFoundationObjC::updateLastImage):
(WebCore::MediaPlayerPrivateAVFoundationObjC::paintWithVideoOutput):
(WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledTrack const):
(WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledAudibleTrack const):
(WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledVisibleTrack const):
LayoutTests:
Reviewed by Eric Carlson.
* http/tests/media/video-hls-copy-into-canvas-expected.txt: Added.
* http/tests/media/video-hls-copy-into-canvas.html: Added.
Modified Paths
Added Paths
Diff
Modified: trunk/LayoutTests/ChangeLog (283584 => 283585)
--- trunk/LayoutTests/ChangeLog 2021-10-05 23:29:57 UTC (rev 283584)
+++ trunk/LayoutTests/ChangeLog 2021-10-05 23:43:27 UTC (rev 283585)
@@ -1,3 +1,14 @@
+2021-10-05 Jean-Yves Avenard <[email protected]>
+
+ createImageBitmap using a HLS video as source always return a black image.
+ https://bugs.webkit.org/show_bug.cgi?id=231225
+ rdar://83884031
+
+ Reviewed by Eric Carlson.
+
+ * http/tests/media/video-hls-copy-into-canvas-expected.txt: Added.
+ * http/tests/media/video-hls-copy-into-canvas.html: Added.
+
2021-10-05 Ayumi Kojima <[email protected]>
[ iOS 15 ] fast/events/ios/rotation/layout-viewport-during-rotation.html is failing.
Added: trunk/LayoutTests/http/tests/media/video-hls-copy-into-canvas-expected.txt (0 => 283585)
--- trunk/LayoutTests/http/tests/media/video-hls-copy-into-canvas-expected.txt (rev 0)
+++ trunk/LayoutTests/http/tests/media/video-hls-copy-into-canvas-expected.txt 2021-10-05 23:43:27 UTC (rev 283585)
@@ -0,0 +1,14 @@
+
+Check that creating an ImageBitmap from a video succeeds.
+
+
+** Set video.src, wait for media data to load
+RUN(video.src = '')
+
+EVENT(canplaythrough)
+EXPECTED (imageBitmap instanceof ImageBitmap == 'true') OK
+EXPECTED (imageBitmap.width != 0 == 'true') OK
+EXPECTED (imageBitmap.height != 0 == 'true') OK
+Image isn't all black \o/
+END OF TEST
+
Added: trunk/LayoutTests/http/tests/media/video-hls-copy-into-canvas.html (0 => 283585)
--- trunk/LayoutTests/http/tests/media/video-hls-copy-into-canvas.html (rev 0)
+++ trunk/LayoutTests/http/tests/media/video-hls-copy-into-canvas.html 2021-10-05 23:43:27 UTC (rev 283585)
@@ -0,0 +1,68 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+
+ <script src=""
+
+ <script>
+ var imageBitmap;
+
+ function getColorIndicesForCoord(x, y, width) {
+ var red = y * (width * 4) + x * 4;
+ return [red, red + 1, red + 2, red + 3];
+ }
+
+ async function canplaythrough()
+ {
+ // As we can't guarantee when an actual image will get painted;
+ // we loop until the condition becomes true: a pixel in the centre is no longer black.
+ let firstRun = true;
+ for (;;) {
+ imageBitmap = await createImageBitmap(video);
+ if (firstRun) {
+ testExpected("imageBitmap instanceof ImageBitmap", true);
+ testExpected("imageBitmap.width != 0", true);
+ testExpected("imageBitmap.height != 0", true);
+ }
+ let ctx = c1.getContext("2d");
+ ctx.drawImage(imageBitmap, 0, 0);
+ const imageData = ctx.getImageData(0, 0, c1.width, c1.height);
+ // At the centre of the video we have a circle, ensure that it's not black.
+ let colorIndices = getColorIndicesForCoord(imageBitmap.width / 2, imageBitmap.height / 2, c1.width);
+ let redIndex = colorIndices[0];
+ let greenIndex = colorIndices[1];
+ let blueIndex = colorIndices[2];
+ let alphaIndex = colorIndices[3];
+
+ let redForCoord = imageData.data[redIndex];
+ let greenForCoord = imageData.data[greenIndex];
+ let blueForCoord = imageData.data[blueIndex];
+ let alphaForCoord = imageData.data[alphaIndex];
+
+ if (redForCoord != 0 && greenForCoord != 0 && blueForCoord != 0 && alphaForCoord == 255) {
+ consoleWrite("Image isn't all black \\o/");
+ endTest();
+ break;
+ }
+ firstRun = false;
+ await sleepFor(100);
+ }
+ }
+
+ function start()
+ {
+ consoleWrite("<br><em>** Set video.src, wait for media data to load</em>");
+ findMediaElement();
+ run("video.src = ''");
+ consoleWrite("");
+ waitForEventOnce("canplaythrough", canplaythrough);
+ }
+ </script>
+ </head>
+ <body _onload_="start()">
+ <video controls></video>
+ <p>Check that creating an ImageBitmap from a video succeeds.</p>
+ <canvas id="c1" width="640" height="480"></canvas>
+ </body>
+</html>
Modified: trunk/Source/WebCore/ChangeLog (283584 => 283585)
--- trunk/Source/WebCore/ChangeLog 2021-10-05 23:29:57 UTC (rev 283584)
+++ trunk/Source/WebCore/ChangeLog 2021-10-05 23:43:27 UTC (rev 283585)
@@ -1,5 +1,32 @@
2021-10-05 Jean-Yves Avenard <[email protected]>
+ createImageBitmap using a HLS video as source always return a black image.
+ https://bugs.webkit.org/show_bug.cgi?id=231225
+ rdar://83884031
+
+ When playing HLS content, [AVURLAsset tracks] return an empty array.
+ We need to instead retrieve it from the AVPlayerItem object. The method
+ paintWithVideoOutput would have bailed out early as a consequence.
+ So we refactor the code a little to retrieve the tracks where they can be found
+
+ Reviewed by Eric Carlson.
+
+ Test: http/tests/media/video-hls-copy-into-canvas.html
+
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h:
+ * platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm:
+ (WebCore::MediaPlayerPrivateAVFoundationObjC::createAVPlayerItem):
+ (WebCore::MediaPlayerPrivateAVFoundationObjC::tracksChanged):
+ (WebCore::MediaPlayerPrivateAVFoundationObjC::updateRotationSession):
+ (WebCore::MediaPlayerPrivateAVFoundationObjC::audioSourceProvider):
+ (WebCore::MediaPlayerPrivateAVFoundationObjC::updateLastImage):
+ (WebCore::MediaPlayerPrivateAVFoundationObjC::paintWithVideoOutput):
+ (WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledTrack const):
+ (WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledAudibleTrack const):
+ (WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledVisibleTrack const):
+
+2021-10-05 Jean-Yves Avenard <[email protected]>
+
REGRESSION (Monterey): paramountplus.com: Cannot enter fullscreen in Safari
https://bugs.webkit.org/show_bug.cgi?id=231005
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h (283584 => 283585)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h 2021-10-05 23:29:57 UTC (rev 283584)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h 2021-10-05 23:43:27 UTC (rev 283585)
@@ -32,6 +32,7 @@
#include <wtf/HashMap.h>
OBJC_CLASS AVAssetImageGenerator;
+OBJC_CLASS AVAssetTrack;
OBJC_CLASS AVAssetResourceLoadingRequest;
OBJC_CLASS AVMediaSelectionGroup;
OBJC_CLASS AVOutputContext;
@@ -50,6 +51,7 @@
typedef struct CGImage *CGImageRef;
typedef struct __CVBuffer *CVPixelBufferRef;
+typedef NSString *AVMediaCharacteristic;
namespace WebCore {
@@ -275,8 +277,9 @@
AVMediaSelectionGroup *safeMediaSelectionGroupForAudibleMedia();
AVMediaSelectionGroup *safeMediaSelectionGroupForVisualMedia();
- NSArray *safeAVAssetTracksForAudibleMedia();
- NSArray *safeAVAssetTracksForVisualMedia();
+ AVAssetTrack* firstEnabledAudibleTrack() const;
+ AVAssetTrack* firstEnabledVisibleTrack() const;
+ AVAssetTrack* firstEnabledTrack(AVMediaCharacteristic) const;
#if ENABLE(DATACUE_VALUE)
void processMetadataTrack();
Modified: trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm (283584 => 283585)
--- trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm 2021-10-05 23:29:57 UTC (rev 283584)
+++ trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm 2021-10-05 23:43:27 UTC (rev 283585)
@@ -247,7 +247,6 @@
static NSArray *itemKVOProperties();
static NSArray *assetTrackMetadataKeyNames();
static NSArray *playerKVOProperties();
-static AVAssetTrack* firstEnabledTrack(NSArray* tracks);
static dispatch_queue_t globalLoaderDelegateQueue()
{
@@ -1168,7 +1167,7 @@
#if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
if (m_provider) {
m_provider->setPlayerItem(m_avPlayerItem.get());
- m_provider->setAudioTrack(firstEnabledTrack(safeAVAssetTracksForAudibleMedia()));
+ m_provider->setAudioTrack(firstEnabledAudibleTrack());
}
#endif
@@ -2086,16 +2085,6 @@
[CATransaction commit];
}
-static AVAssetTrack* firstEnabledTrack(NSArray* tracks)
-{
- NSUInteger index = [tracks indexOfObjectPassingTest:^(id obj, NSUInteger, BOOL *) {
- return [static_cast<AVAssetTrack*>(obj) isEnabled];
- }];
- if (index == NSNotFound)
- return nil;
- return [tracks objectAtIndex:index];
-}
-
void MediaPlayerPrivateAVFoundationObjC::metadataLoaded()
{
MediaPlayerPrivateAVFoundation::metadataLoaded();
@@ -2141,9 +2130,9 @@
if (!m_avPlayerItem) {
// We don't have a player item yet, so check with the asset because some assets support inspection
// prior to becoming ready to play.
- AVAssetTrack* firstEnabledVideoTrack = firstEnabledTrack(safeAVAssetTracksForVisualMedia());
+ auto* firstEnabledVideoTrack = firstEnabledVisibleTrack();
setHasVideo(firstEnabledVideoTrack);
- setHasAudio(firstEnabledTrack(safeAVAssetTracksForAudibleMedia()));
+ setHasAudio(firstEnabledAudibleTrack());
auto size = firstEnabledVideoTrack ? FloatSize(CGSizeApplyAffineTransform([firstEnabledVideoTrack naturalSize], [firstEnabledVideoTrack preferredTransform])) : FloatSize();
// For videos with rotation tag set, the transformation above might return a CGSize instance with negative width or height.
// See https://bugs.webkit.org/show_bug.cgi?id=172648.
@@ -2214,7 +2203,7 @@
#if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
if (m_provider)
- m_provider->setAudioTrack(firstEnabledTrack(safeAVAssetTracksForAudibleMedia()));
+ m_provider->setAudioTrack(firstEnabledAudibleTrack());
#endif
setDelayCharacteristicsChangedNotification(false);
@@ -2227,7 +2216,7 @@
AffineTransform finalTransform = m_avAsset.get().preferredTransform;
FloatSize naturalSize;
- if (auto* firstEnabledVideoTrack = firstEnabledTrack(safeAVAssetTracksForVisualMedia())) {
+ if (auto* firstEnabledVideoTrack = firstEnabledVisibleTrack()) {
naturalSize = FloatSize(firstEnabledVideoTrack.naturalSize);
finalTransform *= firstEnabledVideoTrack.preferredTransform;
}
@@ -2417,7 +2406,7 @@
{
if (!m_provider) {
m_provider = AudioSourceProviderAVFObjC::create(m_avPlayerItem.get());
- m_provider->setAudioTrack(firstEnabledTrack(safeAVAssetTracksForAudibleMedia()));
+ m_provider->setAudioTrack(firstEnabledAudibleTrack());
}
return m_provider.get();
}
@@ -2545,6 +2534,10 @@
if (!m_avPlayerItem || readyState() < MediaPlayer::ReadyState::HaveCurrentData)
return;
+ auto* firstEnabledVideoTrack = firstEnabledVisibleTrack();
+ if (!firstEnabledVideoTrack)
+ return;
+
if (type == UpdateType::UpdateSynchronously && !m_lastImage && !videoOutputHasAvailableFrame())
waitForVideoOutputMediaDataWillChange();
@@ -2572,10 +2565,6 @@
if (!m_lastImage)
return;
- AVAssetTrack* firstEnabledVideoTrack = firstEnabledTrack(safeAVAssetTracksForVisualMedia());
- if (!firstEnabledVideoTrack)
- return;
-
INFO_LOG(LOGIDENTIFIER);
FloatRect imageRect { FloatPoint::zero(), m_lastImage->size() };
@@ -2750,8 +2739,18 @@
}
#endif
-NSArray* MediaPlayerPrivateAVFoundationObjC::safeAVAssetTracksForAudibleMedia()
+AVAssetTrack* MediaPlayerPrivateAVFoundationObjC::firstEnabledTrack(AVMediaCharacteristic characteristic) const
{
+ if (m_avPlayerItem) {
+ for (AVPlayerItemTrack* track in [m_avPlayerItem tracks]) {
+ if (!track.enabled)
+ continue;
+ if (!track.assetTrack)
+ continue;
+ if ([track.assetTrack hasMediaCharacteristic:characteristic])
+ return track.assetTrack;
+ }
+ }
if (!m_avAsset)
return nil;
@@ -2758,18 +2757,24 @@
if ([m_avAsset.get() statusOfValueForKey:@"tracks" error:NULL] != AVKeyValueStatusLoaded)
return nil;
- return [m_avAsset tracksWithMediaCharacteristic:AVMediaCharacteristicAudible];
+ return [] (NSArray* tracks) -> AVAssetTrack* {
+ NSUInteger index = [tracks indexOfObjectPassingTest:^(id obj, NSUInteger, BOOL *) {
+ return [static_cast<AVAssetTrack*>(obj) isEnabled];
+ }];
+ if (index == NSNotFound)
+ return nil;
+ return [tracks objectAtIndex:index];
+ }([m_avAsset tracksWithMediaCharacteristic:characteristic]);
}
-NSArray* MediaPlayerPrivateAVFoundationObjC::safeAVAssetTracksForVisualMedia()
+AVAssetTrack* MediaPlayerPrivateAVFoundationObjC::firstEnabledAudibleTrack() const
{
- if (!m_avAsset)
- return nil;
+ return firstEnabledTrack(AVMediaCharacteristicAudible);
+}
- if ([m_avAsset.get() statusOfValueForKey:@"tracks" error:NULL] != AVKeyValueStatusLoaded)
- return nil;
-
- return [m_avAsset tracksWithMediaCharacteristic:AVMediaCharacteristicVisual];
+AVAssetTrack* MediaPlayerPrivateAVFoundationObjC::firstEnabledVisibleTrack() const
+{
+ return firstEnabledTrack(AVMediaCharacteristicVisual);
}
bool MediaPlayerPrivateAVFoundationObjC::hasLoadedMediaSelectionGroups()
