Diff
Modified: trunk/LayoutTests/ChangeLog (280583 => 280584)
--- trunk/LayoutTests/ChangeLog 2021-08-03 04:18:22 UTC (rev 280583)
+++ trunk/LayoutTests/ChangeLog 2021-08-03 04:21:47 UTC (rev 280584)
@@ -1,3 +1,15 @@
+2021-08-02 Jean-Yves Avenard <[email protected]>
+
+ https://bugs.webkit.org/show_bug.cgi?id=228140
+ rdar://problem/80883882
+
+ Reviewed by Eric Carlson.
+
+ https://bugs.webkit.org/show_bug.cgi?id=228140
+
+ * webaudio/decode-audio-data-webm-opus-expected.txt:
+ * webaudio/decode-audio-data-webm-opus.html:
+
2021-08-02 Chris Dumez <[email protected]>
[COOP] Cross-Origin-Opener-Policy header parsing fails when report-to parameter is present
Modified: trunk/LayoutTests/webaudio/decode-audio-data-webm-opus-expected.txt (280583 => 280584)
--- trunk/LayoutTests/webaudio/decode-audio-data-webm-opus-expected.txt 2021-08-03 04:18:22 UTC (rev 280583)
+++ trunk/LayoutTests/webaudio/decode-audio-data-webm-opus-expected.txt 2021-08-03 04:21:47 UTC (rev 280584)
@@ -4,6 +4,7 @@
PASS Successfully decoded content
+PASS Decoding returned the right number of frames.
PASS successfullyParsed is true
TEST COMPLETE
Modified: trunk/LayoutTests/webaudio/decode-audio-data-webm-opus.html (280583 => 280584)
--- trunk/LayoutTests/webaudio/decode-audio-data-webm-opus.html 2021-08-03 04:18:22 UTC (rev 280583)
+++ trunk/LayoutTests/webaudio/decode-audio-data-webm-opus.html 2021-08-03 04:21:47 UTC (rev 280584)
@@ -10,7 +10,7 @@
window.jsTestIsAsync = true;
-var context = new window.AudioContext();
+var context = new window.AudioContext({ sampleRate: 48000 });
var request = new XMLHttpRequest();
request.open("GET", 'resources/media/opus.webm', true);
request.responseType = "arraybuffer";
@@ -18,6 +18,11 @@
request._onload_ = function() {
context.decodeAudioData(request.response, (buffer) => {
testPassed("Successfully decoded content");
+ // File is exactly 1-0.0065s long @ 48000Hz, so 47688 frames.
+ if (buffer.length === 47688)
+ testPassed("Decoding returned the right number of frames.");
+ else
+ testFailed("Decoding returned the wrong number of frames: " + buffer.length);
finishJSTest();
}, () => {
testFailed("Failed to decode file");
Modified: trunk/Source/WebCore/ChangeLog (280583 => 280584)
--- trunk/Source/WebCore/ChangeLog 2021-08-03 04:18:22 UTC (rev 280583)
+++ trunk/Source/WebCore/ChangeLog 2021-08-03 04:21:47 UTC (rev 280584)
@@ -1,3 +1,40 @@
+2021-08-02 Jean-Yves Avenard <[email protected]>
+
+ [WebAudio] webm; properly trim frames according to the codec delay information
+ https://bugs.webkit.org/show_bug.cgi?id=228140
+ rdar://problem/80883882
+
+ Reviewed by Eric Carlson.
+
+ Add frame-perfect decoding for Opus and Vorbis in WebM container.
+ Unfortunately, due to a vorbis decoder bug, it is not possible to test that
+ vorbis decoding returns the right number of frames.
+ Amend Opus test to ensure the right amount of frames are returned.
+
+ * platform/audio/cocoa/AudioFileReaderCocoa.cpp:
+ (WebCore::AudioFileReader::demuxWebMData const):
+ (WebCore::passthroughInputDataCallback): AudioToolbox internal logging could overflow
+ if we used packet descriptions with offsets, so we take the most commonly used scenario
+ where we have a single packet to decode at a time.
+ (WebCore::AudioFileReader::decodeWebMData const):
+ * platform/graphics/cocoa/AudioTrackPrivateWebM.cpp:
+ (WebCore::AudioTrackPrivateWebM::codecDelay const):
+ (WebCore::AudioTrackPrivateWebM::setDiscardPadding):
+ (WebCore::AudioTrackPrivateWebM::discardPadding const):
+ * platform/graphics/cocoa/AudioTrackPrivateWebM.h:
+ * platform/graphics/cocoa/SourceBufferParserWebM.cpp:
+ (WebCore::SourceBufferParserWebM::OnElementEnd): Fly-by, ensure m_didParseInitializationDataCallback
+ only ever accessed on client thread.
+ (WebCore::SourceBufferParserWebM::OnBlockGroupEnd):
+ * platform/graphics/cocoa/SourceBufferParserWebM.h: Add trimming data callback.
+ Only mark WEBCORE_EXPORT the methods that need it.
+ (WebCore::SourceBufferParserWebM::appendData):
+ (WebCore::SourceBufferParserWebM::setDidParseTrimmingDataCallback):
+ * html/HTMLMediaElement.cpp:
+ (WebCore::HTMLMediaElement::canPlayType const): Make canPlayType(webm/audio; codecs=vorbis)
+ returns maybe if feature flag set.
+
+
2021-08-02 Chris Dumez <[email protected]>
[COOP] Cross-Origin-Opener-Policy header parsing fails when report-to parameter is present
Modified: trunk/Source/WebCore/html/HTMLMediaElement.cpp (280583 => 280584)
--- trunk/Source/WebCore/html/HTMLMediaElement.cpp 2021-08-03 04:18:22 UTC (rev 280583)
+++ trunk/Source/WebCore/html/HTMLMediaElement.cpp 2021-08-03 04:21:47 UTC (rev 280584)
@@ -1070,7 +1070,7 @@
// Temporarily work around bug 226922. For now claim that the opus and vorbis codecs aren't supported
// so that sites relying on this test to determine if webaudio use of opus or vorbis won't error.
auto codecs = contentType.codecs();
- if (support == MediaPlayer::SupportsType::IsSupported && ((codecs.contains("opus") && !webMWebAudioEnabled()) || codecs.contains("vorbis")))
+ if (support == MediaPlayer::SupportsType::IsSupported && ((codecs.contains("opus") || codecs.contains("vorbis")) && !webMWebAudioEnabled()))
support = MediaPlayer::SupportsType::IsNotSupported;
#endif
Modified: trunk/Source/WebCore/platform/audio/cocoa/AudioFileReaderCocoa.cpp (280583 => 280584)
--- trunk/Source/WebCore/platform/audio/cocoa/AudioFileReaderCocoa.cpp 2021-08-03 04:18:22 UTC (rev 280583)
+++ trunk/Source/WebCore/platform/audio/cocoa/AudioFileReaderCocoa.cpp 2021-08-03 04:21:47 UTC (rev 280584)
@@ -35,7 +35,7 @@
#include "AudioBus.h"
#include "AudioFileReader.h"
#include "AudioSampleDataSource.h"
-#include "AudioTrackPrivate.h"
+#include "AudioTrackPrivateWebM.h"
#include "FloatConversion.h"
#include "InbandTextTrackPrivate.h"
#include "Logging.h"
@@ -130,7 +130,7 @@
public:
#if ENABLE(MEDIA_SOURCE)
- SourceBufferParserWebM::InitializationSegment m_initSegment;
+ Ref<AudioTrackPrivateWebM> m_track;
#endif
MediaTime m_duration;
Vector<Ref<MediaSampleAVFObjC>> m_samples;
@@ -184,7 +184,7 @@
bool error = false;
std::optional<uint64_t> audioTrackId;
MediaTime duration;
- SourceBufferParserWebM::InitializationSegment initSegment;
+ RefPtr<AudioTrackPrivateWebM> track;
Vector<Ref<MediaSampleAVFObjC>> samples;
parser->setDidEncounterErrorDuringParsingCallback([&](uint64_t) {
error = true;
@@ -194,7 +194,7 @@
if (audioTrack.track && audioTrack.track->trackUID()) {
duration = init.duration;
audioTrackId = audioTrack.track->trackUID();
- initSegment = WTFMove(init);
+ track = static_pointer_cast<AudioTrackPrivateWebM>(audioTrack.track);
return;
}
}
@@ -207,20 +207,27 @@
parser->setCallOnClientThreadCallback([](auto&& function) {
function();
});
+ parser->setDidParseTrimmingDataCallback([&](uint64_t trackID, const MediaTime& discardPadding) {
+ if (!audioTrackId || !track || trackID != *audioTrackId)
+ return;
+ track->setDiscardPadding(discardPadding);
+ });
SourceBufferParser::Segment segment({ data, dataSize });
parser->appendData(WTFMove(segment));
- if (!audioTrackId)
+ if (!track)
return nullptr;
parser->flushPendingAudioBuffers();
- return makeUnique<AudioFileReaderWebMData>(AudioFileReaderWebMData { WTFMove(initSegment), WTFMove(duration), WTFMove(samples) });
+ return makeUnique<AudioFileReaderWebMData>(AudioFileReaderWebMData { track.releaseNonNull(), WTFMove(duration), WTFMove(samples) });
}
struct PassthroughUserData {
- UInt32 m_channels;
- UInt32 m_dataSize;
- const void* m_data;
+ const UInt32 m_channels;
+ const UInt32 m_dataSize;
+ const char* m_data;
+ const bool m_eos;
+ const Vector<AudioStreamPacketDescription>& m_packets;
UInt32 m_index;
- Vector<AudioStreamPacketDescription>& m_packets;
+ AudioStreamPacketDescription m_packet;
};
// Error value we pass through the decoder to signal that nothing
@@ -236,21 +243,29 @@
auto* userData = static_cast<PassthroughUserData*>(inUserData);
if (userData->m_index == userData->m_packets.size()) {
*numDataPackets = 0;
- return kNoMoreDataErr;
+ return userData->m_eos ? noErr : kNoMoreDataErr;
}
+ if (userData->m_index >= userData->m_packets.size()) {
+ *numDataPackets = 0;
+ return kAudioConverterErr_RequiresPacketDescriptionsError;
+ }
+
if (packetDesc) {
- if (userData->m_index >= userData->m_packets.size()) {
- *numDataPackets = 0;
- return kAudioConverterErr_RequiresPacketDescriptionsError;
- }
- *packetDesc = &userData->m_packets[userData->m_index];
+ userData->m_packet = userData->m_packets[userData->m_index];
+ userData->m_packet.mStartOffset = 0;
+ *packetDesc = &userData->m_packet;
}
data->mBuffers[0].mNumberChannels = userData->m_channels;
- data->mBuffers[0].mDataByteSize = userData->m_dataSize;
- data->mBuffers[0].mData = const_cast<void*>(userData->m_data);
+ data->mBuffers[0].mDataByteSize = userData->m_packets[userData->m_index].mDataByteSize;
+ data->mBuffers[0].mData = const_cast<char*>(userData->m_data + userData->m_packets[userData->m_index].mStartOffset);
+ // Sanity check
+ if (static_cast<char*>(data->mBuffers[0].mData) + data->mBuffers[0].mDataByteSize > userData->m_data + userData->m_dataSize) {
+ RELEASE_LOG_FAULT(WebAudio, "Nonsensical data structure, aborting");
+ return kAudioConverterErr_UnspecifiedError;
+ }
*numDataPackets = 1;
userData->m_index++;
@@ -303,6 +318,12 @@
if (magicCookie && magicCookieSize)
PAL::AudioConverterSetProperty(converter, kAudioConverterDecompressionMagicCookie, magicCookieSize, magicCookie);
+ AudioConverterPrimeInfo primeInfo = { UInt32(m_webmData->m_track->codecDelay().value_or(MediaTime()).toDouble() * outFormat.mSampleRate), 0 };
+ INFO_LOG(LOGIDENTIFIER, "Will drop %u leading frames out of %llu", primeInfo.leadingFrames, numberOfFrames);
+ PAL::AudioConverterSetProperty(converter, kAudioConverterPrimeInfo, sizeof(primeInfo), &primeInfo);
+ UInt32 primeMethod = kConverterPrimeMethod_None;
+ PAL::AudioConverterSetProperty(converter, kAudioConverterPrimeMethod, sizeof(primeMethod), &primeMethod);
+
AudioBufferListHolder decodedBufferList(inFormat.mChannelsPerFrame);
if (!decodedBufferList) {
RELEASE_LOG_FAULT(WebAudio, "Unable to create decoder");
@@ -311,7 +332,8 @@
size_t decodedFrames = 0;
OSStatus status;
- for (auto& sample : m_webmData->m_samples) {
+ for (size_t i = 0; i < m_webmData->m_samples.size(); i++) {
+ auto& sample = m_webmData->m_samples[i];
CMSampleBufferRef sampleBuffer = sample->sampleBuffer();
auto buffer = PAL::CMSampleBufferGetDataBuffer(sampleBuffer);
ASSERT(PAL::CMBlockBufferIsRangeContiguous(buffer, 0, 0));
@@ -331,7 +353,7 @@
if (descriptions.isEmpty())
return { };
- PassthroughUserData userData = { inFormat.mChannelsPerFrame, UInt32(srcSize), srcData, 0, descriptions };
+ PassthroughUserData userData = { inFormat.mChannelsPerFrame, UInt32(srcSize), srcData, i == m_webmData->m_samples.size() - 1, descriptions, 0, { } };
do {
if (numberOfFrames < decodedFrames) {
@@ -353,9 +375,12 @@
return { };
}
decodedFrames += numFrames;
- } while (status != kNoMoreDataErr);
+ } while (status != kNoMoreDataErr && status != noErr);
}
- return decodedFrames;
+ size_t paddingFrames = m_webmData->m_track->discardPadding().value_or(MediaTime()).toDouble() * outFormat.mSampleRate;
+ if (decodedFrames > paddingFrames)
+ return decodedFrames - paddingFrames;
+ return 0;
}
#endif
Modified: trunk/Source/WebCore/platform/graphics/cocoa/AudioTrackPrivateWebM.cpp (280583 => 280584)
--- trunk/Source/WebCore/platform/graphics/cocoa/AudioTrackPrivateWebM.cpp 2021-08-03 04:18:22 UTC (rev 280583)
+++ trunk/Source/WebCore/platform/graphics/cocoa/AudioTrackPrivateWebM.cpp 2021-08-03 04:21:47 UTC (rev 280584)
@@ -86,6 +86,26 @@
return 0;
}
+std::optional<MediaTime> AudioTrackPrivateWebM::codecDelay() const
+{
+ if (!m_track.codec_delay.is_present())
+ return { };
+ constexpr uint32_t k_us_in_seconds = 1000000000;
+ return MediaTime(m_track.codec_delay.value(), k_us_in_seconds);
}
+void AudioTrackPrivateWebM::setDiscardPadding(const MediaTime& discardPadding)
+{
+ m_discardPadding = discardPadding;
+}
+
+std::optional<MediaTime> AudioTrackPrivateWebM::discardPadding() const
+{
+ if (m_discardPadding.isInvalid() || m_discardPadding < MediaTime())
+ return { };
+ return m_discardPadding;
+}
+
+}
+
#endif // ENABLE(MEDIA_SOURCE)
Modified: trunk/Source/WebCore/platform/graphics/cocoa/AudioTrackPrivateWebM.h (280583 => 280584)
--- trunk/Source/WebCore/platform/graphics/cocoa/AudioTrackPrivateWebM.h 2021-08-03 04:18:22 UTC (rev 280583)
+++ trunk/Source/WebCore/platform/graphics/cocoa/AudioTrackPrivateWebM.h 2021-08-03 04:21:47 UTC (rev 280584)
@@ -43,10 +43,14 @@
int trackIndex() const final;
std::optional<uint64_t> trackUID() const final;
std::optional<bool> defaultEnabled() const final;
+ std::optional<MediaTime> codecDelay() const;
+ void setDiscardPadding(const MediaTime&);
+ std::optional<MediaTime> discardPadding() const;
private:
AudioTrackPrivateWebM(webm::TrackEntry&&);
webm::TrackEntry m_track;
+ MediaTime m_discardPadding { MediaTime::invalidTime() };
mutable AtomString m_trackID;
mutable AtomString m_label;
mutable AtomString m_language;
Modified: trunk/Source/WebCore/platform/graphics/cocoa/SourceBufferParserWebM.cpp (280583 => 280584)
--- trunk/Source/WebCore/platform/graphics/cocoa/SourceBufferParserWebM.cpp 2021-08-03 04:18:22 UTC (rev 280583)
+++ trunk/Source/WebCore/platform/graphics/cocoa/SourceBufferParserWebM.cpp 2021-08-03 04:21:47 UTC (rev 280584)
@@ -264,6 +264,8 @@
// FIXME: Remove this once kCMVideoCodecType_VP9 is added to CMFormatDescription.h
constexpr CMVideoCodecType kCMVideoCodecType_VP9 { 'vp09' };
+constexpr uint32_t k_us_in_seconds = 1000000000;
+
static bool isWebmParserAvailable()
{
return !!webm::swap;
@@ -779,9 +781,10 @@
return Status(Status::Code(ErrorCode::ContentEncrypted));
}
- if (m_initializationSegmentEncountered && m_didParseInitializationDataCallback) {
+ if (m_initializationSegmentEncountered) {
m_callOnClientThreadCallback([this, protectedThis = makeRef(*this), initializationSegment = WTFMove(*m_initializationSegment)]() mutable {
- m_didParseInitializationDataCallback(WTFMove(initializationSegment));
+ if (m_didParseInitializationDataCallback)
+ m_didParseInitializationDataCallback(WTFMove(initializationSegment));
});
}
m_initializationSegmentEncountered = false;
@@ -847,7 +850,7 @@
}
auto timecodeScale = info.timecode_scale.is_present() ? info.timecode_scale.value() : 1000000;
- m_timescale = 1000000000 / timecodeScale;
+ m_timescale = k_us_in_seconds / timecodeScale;
m_initializationSegment->duration = info.duration.is_present() ? MediaTime(info.duration.value(), m_timescale) : MediaTime::indefiniteTime();
return Status(Status::kOkCompleted);
@@ -1023,8 +1026,21 @@
webm::Status SourceBufferParserWebM::OnBlockGroupEnd(const webm::ElementMetadata& metadata, const webm::BlockGroup& blockGroup)
{
UNUSED_PARAM(metadata);
- UNUSED_PARAM(blockGroup);
INFO_LOG_IF_POSSIBLE(LOGIDENTIFIER);
+ if (blockGroup.block.is_present() && blockGroup.discard_padding.is_present()) {
+ auto trackNumber = blockGroup.block.value().track_number;
+ auto* trackData = trackDataForTrackNumber(trackNumber);
+ if (!trackData) {
+ ERROR_LOG_IF_POSSIBLE(LOGIDENTIFIER, "Ignoring unknown track number ", trackNumber);
+ return Status(Status::kOkCompleted);
+ }
+ if (trackData->track().track_uid.is_present() && blockGroup.discard_padding.value() > 0) {
+ m_callOnClientThreadCallback([this, protectedThis = makeRef(*this), trackID = trackData->track().track_uid.value(), padding = MediaTime(blockGroup.discard_padding.value(), k_us_in_seconds)]() {
+ if (m_didParseTrimmingDataCallback)
+ m_didParseTrimmingDataCallback(trackID, padding);
+ });
+ }
+ }
return Status(Status::kOkCompleted);
}
@@ -1212,7 +1228,7 @@
uint64_t duration = 0;
if (track.default_duration.is_present())
- duration = track.default_duration.value() * presentationTime.timescale / 1000000000;
+ duration = track.default_duration.value() * presentationTime.timescale / k_us_in_seconds;
CMSampleBufferRef rawSampleBuffer = nullptr;
size_t frameSize = PAL::CMBlockBufferGetDataLength(m_currentBlockBuffer.get());
Modified: trunk/Source/WebCore/platform/graphics/cocoa/SourceBufferParserWebM.h (280583 => 280584)
--- trunk/Source/WebCore/platform/graphics/cocoa/SourceBufferParserWebM.h 2021-08-03 04:18:22 UTC (rev 280583)
+++ trunk/Source/WebCore/platform/graphics/cocoa/SourceBufferParserWebM.h 2021-08-03 04:21:47 UTC (rev 280584)
@@ -55,7 +55,7 @@
class MediaSampleAVFObjC;
-class WEBCORE_EXPORT SourceBufferParserWebM : public SourceBufferParser, private webm::Callback {
+class SourceBufferParserWebM : public SourceBufferParser, private webm::Callback {
WTF_MAKE_FAST_ALLOCATED;
public:
class StreamingVectorReader;
@@ -63,7 +63,7 @@
static bool isWebMFormatReaderAvailable();
static MediaPlayerEnums::SupportsType isContentTypeSupported(const ContentType&);
static const HashSet<String, ASCIICaseInsensitiveHash>& webmMIMETypes();
- static RefPtr<SourceBufferParserWebM> create(const ContentType&);
+ WEBCORE_EXPORT static RefPtr<SourceBufferParserWebM> create(const ContentType&);
SourceBufferParserWebM();
~SourceBufferParserWebM();
@@ -73,7 +73,7 @@
const webm::Status& status() const { return m_status; }
Type type() const { return Type::WebM; }
- void appendData(Segment&&, CompletionHandler<void()>&& = [] { }, AppendFlags = AppendFlags::None) final;
+ WEBCORE_EXPORT void appendData(Segment&&, CompletionHandler<void()>&& = [] { }, AppendFlags = AppendFlags::None) final;
void flushPendingMediaData() final;
void setShouldProvideMediaDataForTrackID(bool, uint64_t) final;
bool shouldProvideMediadataForTrackID(uint64_t) final;
@@ -83,9 +83,14 @@
void flushPendingAudioBuffers();
void setMinimumAudioSampleDuration(float);
- void setLogger(const WTF::Logger&, const void* identifier) final;
+ WEBCORE_EXPORT void setLogger(const WTF::Logger&, const void* identifier) final;
void provideMediaData(RetainPtr<CMSampleBufferRef>, uint64_t, std::optional<size_t> byteRangeOffset);
+ using DidParseTrimmingDataCallback = WTF::Function<void(uint64_t trackID, const MediaTime& discardPadding)>;
+ void setDidParseTrimmingDataCallback(DidParseTrimmingDataCallback&& callback)
+ {
+ m_didParseTrimmingDataCallback = WTFMove(callback);
+ }
enum class ErrorCode : int32_t {
SourceBufferParserWebMErrorCodeStart = 2000,
@@ -287,6 +292,7 @@
RefPtr<const WTF::Logger> m_logger;
const void* m_logIdentifier { nullptr };
uint64_t m_nextChildIdentifier { 0 };
+ DidParseTrimmingDataCallback m_didParseTrimmingDataCallback;
};
}
