Diff
Modified: branches/safari-609-branch/LayoutTests/ChangeLog (255024 => 255025)
--- branches/safari-609-branch/LayoutTests/ChangeLog 2020-01-23 21:44:21 UTC (rev 255024)
+++ branches/safari-609-branch/LayoutTests/ChangeLog 2020-01-23 21:44:24 UTC (rev 255025)
@@ -1,5 +1,58 @@
2020-01-23 Russell Epstein <[email protected]>
+ Cherry-pick r254761. rdar://problem/58807932
+
+ [MSE] Decode glitches when watching videos on CNN.com
+ https://bugs.webkit.org/show_bug.cgi?id=206412
+ <rdar://problem/55685630>
+
+ Reviewed by Xabier Rodriguez-Calvar.
+
+ Source/WebCore:
+
+ Test: media/media-source/media-source-samples-out-of-order.html
+
+ The "Coded frame processing" algorithm has a known shortcoming <https://github.com/w3c/media-source/issues/187>
+ when dealing appends of with "SAP Type 2" content, or in general terms, appending data where the resulting samples
+ have presentation times that do not increase monotonically. When this occurs, the ordering of samples in presentation
+ time will be different from the ordering of samples in decode time. The decoder requires samples to be enqueued in
+ decode time order, but the MSE specification only checks for overlapping samples in presentation time order. During
+ appends of out-of-order samples, this can lead to new samples being inserted between a previously appended sample and
+ the sample on which that sample depends.
+
+ To resolve this, add a new step in the implementation of the "coded frame processing" algorithm in
+ SourceBuffer::sourceBufferPrivateDidReceiveSample(). When the incoming frame is a sync sample, search forward
+ in the TrackBuffer for all previous samples in between the new sync sample, and the next sync sample. All the
+ samples found in this step would fail to decode correctly if enqueued after the new (possibly different resolution)
+ sync sample, so they are removed in this step.
+
+ * Modules/mediasource/SampleMap.cpp:
+ (WebCore::DecodeOrderSampleMap::findSampleAfterDecodeKey):
+ * Modules/mediasource/SampleMap.h:
+ * Modules/mediasource/SourceBuffer.cpp:
+ (WebCore::SourceBuffer::sourceBufferPrivateDidReceiveSample):
+
+ LayoutTests:
+
+ * media/media-source/media-source-samples-out-of-order-expected.txt: Added.
+ * media/media-source/media-source-samples-out-of-order.html: Added.
+
+
+ git-svn-id: https://svn.webkit.org/repository/webkit/trunk@254761 268f45cc-cd09-0410-ab3c-d52691b4dbfc
+
+ 2020-01-17 Jer Noble <[email protected]>
+
+ [MSE] Decode glitches when watching videos on CNN.com
+ https://bugs.webkit.org/show_bug.cgi?id=206412
+ <rdar://problem/55685630>
+
+ Reviewed by Xabier Rodriguez-Calvar.
+
+ * media/media-source/media-source-samples-out-of-order-expected.txt: Added.
+ * media/media-source/media-source-samples-out-of-order.html: Added.
+
+2020-01-23 Russell Epstein <[email protected]>
+
Cherry-pick r254722. rdar://problem/58811423
REGRESSION (r251110): Crash on https://developer.apple.com/tutorials/swiftui/creating-and-combining-views
Added: branches/safari-609-branch/LayoutTests/media/media-source/media-source-samples-out-of-order-expected.txt (0 => 255025)
--- branches/safari-609-branch/LayoutTests/media/media-source/media-source-samples-out-of-order-expected.txt (rev 0)
+++ branches/safari-609-branch/LayoutTests/media/media-source/media-source-samples-out-of-order-expected.txt 2020-01-23 21:44:24 UTC (rev 255025)
@@ -0,0 +1,20 @@
+
+RUN(video.src = ""
+EVENT(sourceopen)
+RUN(sourceBuffer = source.addSourceBuffer("video/mock; codecs=mock"))
+First segment has normal, monotonically increasing samples.
+RUN(sourceBuffer.appendBuffer(mediaSegment))
+EVENT(updateend)
+EXPECTED (bufferedSamples.length == '3') OK
+{PTS({1/1 = 1.000000}), DTS({1/1 = 1.000000}), duration({1/1 = 1.000000}), flags(1), generation(0)}
+{PTS({2/1 = 2.000000}), DTS({2/1 = 2.000000}), duration({1/1 = 1.000000}), flags(0), generation(0)}
+{PTS({3/1 = 3.000000}), DTS({3/1 = 3.000000}), duration({1/1 = 1.000000}), flags(0), generation(0)}
+Second, overlapping segment has out-of-display-order samples. This append should replace the last sample from the previous append.
+RUN(sourceBuffer.appendBuffer(mediaSegment))
+EVENT(updateend)
+EXPECTED (bufferedSamples.length == '3') OK
+{PTS({1/1 = 1.000000}), DTS({1/1 = 1.000000}), duration({1/1 = 1.000000}), flags(1), generation(0)}
+{PTS({2/1 = 2.000000}), DTS({2/1 = 2.000000}), duration({1/1 = 1.000000}), flags(0), generation(0)}
+{PTS({4/1 = 4.000000}), DTS({2/1 = 2.000000}), duration({1/1 = 1.000000}), flags(1), generation(1)}
+END OF TEST
+
Added: branches/safari-609-branch/LayoutTests/media/media-source/media-source-samples-out-of-order.html (0 => 255025)
--- branches/safari-609-branch/LayoutTests/media/media-source/media-source-samples-out-of-order.html (rev 0)
+++ branches/safari-609-branch/LayoutTests/media/media-source/media-source-samples-out-of-order.html 2020-01-23 21:44:24 UTC (rev 255025)
@@ -0,0 +1,57 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <title>media-source-samples-out-of-order</title>
+ <script src=""
+ <script src=""
+ <script>
+ var source;
+ var sourceBuffer;
+ var initSegment;
+ var mediaSegment;
+
+ if (window.internals)
+ internals.initializeMockMediaSource();
+
+ window.addEventListener('load', async event => {
+ findMediaElement();
+
+ source = new MediaSource();
+ run('video.src = ""
+ await waitFor(source, 'sourceopen');
+
+ run('sourceBuffer = source.addSourceBuffer("video/mock; codecs=mock")');
+ consoleWrite('First segment has normal, monotonically increasing samples.')
+ mediaSegment = concatenateSamples([
+ makeAInit(2, [makeATrack(1, 'mock', TRACK_KIND.AUDIO)]),
+ makeASample(1, 1, 1, 1, 1, SAMPLE_FLAG.SYNC, 0),
+ makeASample(2, 2, 1, 1, 1, SAMPLE_FLAG.NONE, 0),
+ makeASample(3, 3, 1, 1, 1, SAMPLE_FLAG.NONE, 0),
+ ]);
+ run('sourceBuffer.appendBuffer(mediaSegment)');
+ await waitFor(sourceBuffer, 'updateend');
+
+ bufferedSamples = internals.bufferedSamplesForTrackID(sourceBuffer, 1);
+ testExpected("bufferedSamples.length", 3);
+ bufferedSamples.forEach(consoleWrite);
+
+ consoleWrite('Second, overlapping segment has out-of-display-order samples. This append should replace the last sample from the previous append.')
+ mediaSegment = concatenateSamples([
+ makeAInit(2, [makeATrack(1, 'mock', TRACK_KIND.AUDIO)]),
+ makeASample(4, 2, 1, 1, 1, SAMPLE_FLAG.SYNC, 1),
+ ]);
+ run('sourceBuffer.appendBuffer(mediaSegment)');
+ await waitFor(sourceBuffer, 'updateend');
+
+ bufferedSamples = internals.bufferedSamplesForTrackID(sourceBuffer, 1);
+ testExpected("bufferedSamples.length", 3);
+ bufferedSamples.forEach(consoleWrite);
+
+ endTest();
+ });
+ </script>
+</head>
+<body>
+ <video></video>
+</body>
+</html>
Modified: branches/safari-609-branch/Source/WebCore/ChangeLog (255024 => 255025)
--- branches/safari-609-branch/Source/WebCore/ChangeLog 2020-01-23 21:44:21 UTC (rev 255024)
+++ branches/safari-609-branch/Source/WebCore/ChangeLog 2020-01-23 21:44:24 UTC (rev 255025)
@@ -1,5 +1,77 @@
2020-01-23 Russell Epstein <[email protected]>
+ Cherry-pick r254761. rdar://problem/58807932
+
+ [MSE] Decode glitches when watching videos on CNN.com
+ https://bugs.webkit.org/show_bug.cgi?id=206412
+ <rdar://problem/55685630>
+
+ Reviewed by Xabier Rodriguez-Calvar.
+
+ Source/WebCore:
+
+ Test: media/media-source/media-source-samples-out-of-order.html
+
+ The "Coded frame processing" algorithm has a known shortcoming <https://github.com/w3c/media-source/issues/187>
+ when dealing appends of with "SAP Type 2" content, or in general terms, appending data where the resulting samples
+ have presentation times that do not increase monotonically. When this occurs, the ordering of samples in presentation
+ time will be different from the ordering of samples in decode time. The decoder requires samples to be enqueued in
+ decode time order, but the MSE specification only checks for overlapping samples in presentation time order. During
+ appends of out-of-order samples, this can lead to new samples being inserted between a previously appended sample and
+ the sample on which that sample depends.
+
+ To resolve this, add a new step in the implementation of the "coded frame processing" algorithm in
+ SourceBuffer::sourceBufferPrivateDidReceiveSample(). When the incoming frame is a sync sample, search forward
+ in the TrackBuffer for all previous samples in between the new sync sample, and the next sync sample. All the
+ samples found in this step would fail to decode correctly if enqueued after the new (possibly different resolution)
+ sync sample, so they are removed in this step.
+
+ * Modules/mediasource/SampleMap.cpp:
+ (WebCore::DecodeOrderSampleMap::findSampleAfterDecodeKey):
+ * Modules/mediasource/SampleMap.h:
+ * Modules/mediasource/SourceBuffer.cpp:
+ (WebCore::SourceBuffer::sourceBufferPrivateDidReceiveSample):
+
+ LayoutTests:
+
+ * media/media-source/media-source-samples-out-of-order-expected.txt: Added.
+ * media/media-source/media-source-samples-out-of-order.html: Added.
+
+
+ git-svn-id: https://svn.webkit.org/repository/webkit/trunk@254761 268f45cc-cd09-0410-ab3c-d52691b4dbfc
+
+ 2020-01-17 Jer Noble <[email protected]>
+
+ [MSE] Decode glitches when watching videos on CNN.com
+ https://bugs.webkit.org/show_bug.cgi?id=206412
+ <rdar://problem/55685630>
+
+ Reviewed by Xabier Rodriguez-Calvar.
+
+ Test: media/media-source/media-source-samples-out-of-order.html
+
+ The "Coded frame processing" algorithm has a known shortcoming <https://github.com/w3c/media-source/issues/187>
+ when dealing appends of with "SAP Type 2" content, or in general terms, appending data where the resulting samples
+ have presentation times that do not increase monotonically. When this occurs, the ordering of samples in presentation
+ time will be different from the ordering of samples in decode time. The decoder requires samples to be enqueued in
+ decode time order, but the MSE specification only checks for overlapping samples in presentation time order. During
+ appends of out-of-order samples, this can lead to new samples being inserted between a previously appended sample and
+ the sample on which that sample depends.
+
+ To resolve this, add a new step in the implementation of the "coded frame processing" algorithm in
+ SourceBuffer::sourceBufferPrivateDidReceiveSample(). When the incoming frame is a sync sample, search forward
+ in the TrackBuffer for all previous samples in between the new sync sample, and the next sync sample. All the
+ samples found in this step would fail to decode correctly if enqueued after the new (possibly different resolution)
+ sync sample, so they are removed in this step.
+
+ * Modules/mediasource/SampleMap.cpp:
+ (WebCore::DecodeOrderSampleMap::findSampleAfterDecodeKey):
+ * Modules/mediasource/SampleMap.h:
+ * Modules/mediasource/SourceBuffer.cpp:
+ (WebCore::SourceBuffer::sourceBufferPrivateDidReceiveSample):
+
+2020-01-23 Russell Epstein <[email protected]>
+
Cherry-pick r254722. rdar://problem/58811423
REGRESSION (r251110): Crash on https://developer.apple.com/tutorials/swiftui/creating-and-combining-views
Modified: branches/safari-609-branch/Source/WebCore/Modules/mediasource/SampleMap.cpp (255024 => 255025)
--- branches/safari-609-branch/Source/WebCore/Modules/mediasource/SampleMap.cpp 2020-01-23 21:44:21 UTC (rev 255024)
+++ branches/safari-609-branch/Source/WebCore/Modules/mediasource/SampleMap.cpp 2020-01-23 21:44:24 UTC (rev 255025)
@@ -188,6 +188,11 @@
return m_samples.find(key);
}
+DecodeOrderSampleMap::iterator DecodeOrderSampleMap::findSampleAfterDecodeKey(const KeyType& key)
+{
+ return m_samples.upper_bound(key);
+}
+
PresentationOrderSampleMap::reverse_iterator PresentationOrderSampleMap::reverseFindSampleContainingPresentationTime(const MediaTime& time)
{
auto range = std::equal_range(rbegin(), rend(), time, SampleIsGreaterThanMediaTimeComparator<MapType>());
Modified: branches/safari-609-branch/Source/WebCore/Modules/mediasource/SampleMap.h (255024 => 255025)
--- branches/safari-609-branch/Source/WebCore/Modules/mediasource/SampleMap.h 2020-01-23 21:44:21 UTC (rev 255024)
+++ branches/safari-609-branch/Source/WebCore/Modules/mediasource/SampleMap.h 2020-01-23 21:44:24 UTC (rev 255025)
@@ -93,6 +93,7 @@
const_reverse_iterator rend() const { return m_samples.rend(); }
WEBCORE_EXPORT iterator findSampleWithDecodeKey(const KeyType&);
+ WEBCORE_EXPORT iterator findSampleAfterDecodeKey(const KeyType&);
WEBCORE_EXPORT reverse_iterator reverseFindSampleWithDecodeKey(const KeyType&);
WEBCORE_EXPORT reverse_iterator findSyncSamplePriorToPresentationTime(const MediaTime&, const MediaTime& threshold = MediaTime::positiveInfiniteTime());
WEBCORE_EXPORT reverse_iterator findSyncSamplePriorToDecodeIterator(reverse_iterator);
Modified: branches/safari-609-branch/Source/WebCore/Modules/mediasource/SourceBuffer.cpp (255024 => 255025)
--- branches/safari-609-branch/Source/WebCore/Modules/mediasource/SourceBuffer.cpp 2020-01-23 21:44:21 UTC (rev 255024)
+++ branches/safari-609-branch/Source/WebCore/Modules/mediasource/SourceBuffer.cpp 2020-01-23 21:44:24 UTC (rev 255025)
@@ -1677,6 +1677,29 @@
erasedSamples.addRange(iter_pair.first, iter_pair.second);
}
+ // When appending media containing B-frames (media whose samples' presentation timestamps
+ // do not increase monotonically, the prior erase steps could leave a sample in the trackBuffer
+ // which will be disconnected from its previous I-frame. If the incoming frame is an I-frame,
+ // remove all samples in decode order between the incoming I-frame's decode timestamp and the
+ // next I-frame. See <https://github.com/w3c/media-source/issues/187> for a discussion of what
+ // the how the MSE specification should handlie this secnario.
+ do {
+ if (!sample.isSync())
+ break;
+
+ DecodeOrderSampleMap::KeyType decodeKey(sample.decodeTime(), sample.presentationTime());
+ auto nextSampleInDecodeOrder = trackBuffer.samples.decodeOrder().findSampleAfterDecodeKey(decodeKey);
+ if (nextSampleInDecodeOrder == trackBuffer.samples.decodeOrder().end())
+ break;
+
+ if (nextSampleInDecodeOrder->second->isSync())
+ break;
+
+ auto nextSyncSample = trackBuffer.samples.decodeOrder().findSyncSampleAfterDecodeIterator(nextSampleInDecodeOrder);
+ INFO_LOG(LOGIDENTIFIER, "Discovered out-of-order frames, from: ", *nextSampleInDecodeOrder->second, " to: ", (nextSyncSample == trackBuffer.samples.decodeOrder().end() ? "[end]"_s : toString(*nextSyncSample->second)));
+ erasedSamples.addRange(nextSampleInDecodeOrder, nextSyncSample);
+ } while (false);
+
// There are many files out there where the frame times are not perfectly contiguous and may have small overlaps
// between the beginning of a frame and the end of the previous one; therefore a tolerance is needed whenever
// durations are considered.
