Diff
Modified: trunk/Source/WebCore/ChangeLog (274837 => 274838)
--- trunk/Source/WebCore/ChangeLog 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebCore/ChangeLog 2021-03-23 01:53:56 UTC (rev 274838)
@@ -1,3 +1,46 @@
+2021-03-22 Myles C. Maxfield <[email protected]>
+
+ [GPU Process]: Improve getImageData() perf part 1: Add a GetImageData display list item
+ https://bugs.webkit.org/show_bug.cgi?id=223544
+
+ Reviewed by Wenson Hsieh.
+
+ We're moving to a design for getImageData() where the web process and the GPU process
+ set up a shmem + semaphore ahead-of-time, so that when we encounter a call to getImageData(),
+ the GPU process can just write into the shmem and signal the semaphore. The web process just
+ encodes the getImageData() call into the regular display list, flushAsync()s to kick the GPU
+ process, then just waits on the semaphore. Initial performance testing shows that this is
+ way, way faster than sending messages containing pixel color data payloads.
+
+ This patch is just the first step: Adding a GetImageData display list item. The item
+ currently gets recorded in the web process and applied in the GPU process, but the application
+ process currently does nothing.
+
+ No new tests because there is no behavior change yet.
+
+ * platform/graphics/displaylists/DisplayList.cpp:
+ (WebCore::DisplayList::DisplayList::append):
+ * platform/graphics/displaylists/DisplayListItemBuffer.cpp:
+ (WebCore::DisplayList::ItemHandle::apply):
+ (WebCore::DisplayList::ItemHandle::destroy):
+ (WebCore::DisplayList::ItemHandle::safeCopy const):
+ * platform/graphics/displaylists/DisplayListItemBuffer.h:
+ * platform/graphics/displaylists/DisplayListItemType.cpp:
+ (WebCore::DisplayList::sizeOfItemInBytes):
+ (WebCore::DisplayList::isDrawingItem):
+ (WebCore::DisplayList::isInlineItem):
+ * platform/graphics/displaylists/DisplayListItemType.h:
+ * platform/graphics/displaylists/DisplayListItems.cpp:
+ (WebCore::DisplayList::operator<<):
+ (WebCore::DisplayList::PutImageData::apply const): Deleted.
+ * platform/graphics/displaylists/DisplayListItems.h:
+ (WebCore::DisplayList::GetImageData::GetImageData):
+ (WebCore::DisplayList::GetImageData::outputFormat const):
+ (WebCore::DisplayList::GetImageData::srcRect const):
+ * platform/graphics/displaylists/DisplayListRecorder.cpp:
+ (WebCore::DisplayList::Recorder::getImageData):
+ * platform/graphics/displaylists/DisplayListRecorder.h:
+
2021-03-22 Sam Weinig <[email protected]>
[WebIDL] Remove the need to specify [MayThrowException]
Modified: trunk/Source/WebCore/platform/graphics/displaylists/DisplayList.cpp (274837 => 274838)
--- trunk/Source/WebCore/platform/graphics/displaylists/DisplayList.cpp 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebCore/platform/graphics/displaylists/DisplayList.cpp 2021-03-23 01:53:56 UTC (rev 274838)
@@ -287,6 +287,8 @@
return append<MetaCommandChangeDestinationImageBuffer>(item.get<MetaCommandChangeDestinationImageBuffer>());
case ItemType::MetaCommandChangeItemBuffer:
return append<MetaCommandChangeItemBuffer>(item.get<MetaCommandChangeItemBuffer>());
+ case ItemType::GetImageData:
+ return append<GetImageData>(item.get<GetImageData>());
case ItemType::PutImageData:
return append<PutImageData>(item.get<PutImageData>());
case ItemType::PaintFrameForMedia:
Modified: trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemBuffer.cpp (274837 => 274838)
--- trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemBuffer.cpp 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemBuffer.cpp 2021-03-23 01:53:56 UTC (rev 274838)
@@ -187,8 +187,10 @@
case ItemType::MetaCommandChangeDestinationImageBuffer:
case ItemType::MetaCommandChangeItemBuffer:
return;
+ case ItemType::GetImageData:
case ItemType::PutImageData:
- get<PutImageData>().apply(context);
+ // Should already be handled by the delegate.
+ ASSERT_NOT_REACHED();
return;
case ItemType::PaintFrameForMedia:
get<PaintFrameForMedia>().apply(context);
@@ -275,6 +277,9 @@
case ItemType::FillRoundedRect:
get<FillRoundedRect>().~FillRoundedRect();
return;
+ case ItemType::GetImageData:
+ static_assert(std::is_trivially_destructible<GetImageData>::value);
+ return;
case ItemType::PutImageData:
get<PutImageData>().~PutImageData();
return;
@@ -483,6 +488,8 @@
return copyInto<FillRectWithRoundedHole>(*this, itemOffset);
case ItemType::FillRoundedRect:
return copyInto<FillRoundedRect>(*this, itemOffset);
+ case ItemType::GetImageData:
+ return copyInto<GetImageData>(*this, itemOffset);
case ItemType::PutImageData:
return copyInto<PutImageData>(*this, itemOffset);
case ItemType::SetLineDash:
Modified: trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemBuffer.h (274837 => 274838)
--- trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemBuffer.h 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemBuffer.h 2021-03-23 01:53:56 UTC (rev 274838)
@@ -139,7 +139,7 @@
// writable buffer handle; if remaining buffer capacity is insufficient to store the item, a
// new buffer will be allocated (either by the ItemBufferWritingClient, if set, or by the item
// buffer itself if there is no client). Items are placed back-to-back in these item buffers,
- // with padding after each item to ensure that all items are aligned to 8 bytes.
+ // with padding between each item to ensure that all items are aligned to 8 bytes.
//
// If a writing client is present and requires custom encoding for the given item type T, the
// item buffer will ask the client for an opaque SharedBuffer containing encoded data for the
Modified: trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemType.cpp (274837 => 274838)
--- trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemType.cpp 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemType.cpp 2021-03-23 01:53:56 UTC (rev 274838)
@@ -132,6 +132,8 @@
return sizeof(MetaCommandChangeDestinationImageBuffer);
case ItemType::MetaCommandChangeItemBuffer:
return sizeof(MetaCommandChangeItemBuffer);
+ case ItemType::GetImageData:
+ return sizeof(GetImageData);
case ItemType::PutImageData:
return sizeof(PutImageData);
case ItemType::PaintFrameForMedia:
@@ -169,6 +171,9 @@
bool isDrawingItem(ItemType type)
{
+ /* See the comment at the top of DisplayListItems.h for what this means.
+ * This needs to match all the "static constexpr bool isDrawingItem"s inside the individual item classes. */
+
switch (type) {
case ItemType::ApplyDeviceScaleFactor:
#if USE(CG)
@@ -202,6 +207,7 @@
case ItemType::SetState:
case ItemType::SetStrokeThickness:
case ItemType::Translate:
+ case ItemType::GetImageData:
return false;
case ItemType::BeginTransparencyLayer:
case ItemType::ClearRect:
@@ -251,6 +257,10 @@
bool isInlineItem(ItemType type)
{
+ /* See the comment at the top of DisplayListItems.h for what this means.
+ * This needs to match (1) RemoteImageBufferProxy::encodeItem(), (2) RemoteRenderingBackend::decodeItem(),
+ * and (3) all the "static constexpr bool isInlineItem"s inside the individual item classes. */
+
switch (type) {
case ItemType::ClipOutToPath:
case ItemType::ClipPath:
@@ -320,6 +330,7 @@
case ItemType::StrokeRect:
case ItemType::StrokeLine:
case ItemType::Translate:
+ case ItemType::GetImageData:
return true;
}
ASSERT_NOT_REACHED();
Modified: trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemType.h (274837 => 274838)
--- trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemType.h 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItemType.h 2021-03-23 01:53:56 UTC (rev 274838)
@@ -79,6 +79,7 @@
FlushContext,
MetaCommandChangeDestinationImageBuffer,
MetaCommandChangeItemBuffer,
+ GetImageData,
PutImageData,
PaintFrameForMedia,
StrokeRect,
Modified: trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItems.cpp (274837 => 274838)
--- trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItems.cpp 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItems.cpp 2021-03-23 01:53:56 UTC (rev 274838)
@@ -769,6 +769,13 @@
return ts;
}
+static TextStream& operator<<(TextStream& ts, const GetImageData& item)
+{
+ ts.dumpProperty("outputFormat", item.outputFormat());
+ ts.dumpProperty("srcRect", item.srcRect());
+ return ts;
+}
+
PutImageData::PutImageData(AlphaPremultiplication inputFormat, const ImageData& imageData, const IntRect& srcRect, const IntPoint& destPoint, AlphaPremultiplication destFormat)
: m_srcRect(srcRect)
, m_destPoint(destPoint)
@@ -787,12 +794,6 @@
{
}
-NO_RETURN_DUE_TO_ASSERT void PutImageData::apply(GraphicsContext&) const
-{
- // Should be handled by the delegate.
- ASSERT_NOT_REACHED();
-}
-
static TextStream& operator<<(TextStream& ts, const PutImageData& item)
{
ts.dumpProperty("inputFormat", item.inputFormat());
@@ -1061,6 +1062,7 @@
case ItemType::FlushContext: ts << "flush-context"; break;
case ItemType::MetaCommandChangeDestinationImageBuffer: ts << "meta-command-change-destination-image-buffer"; break;
case ItemType::MetaCommandChangeItemBuffer: ts << "meta-command-change-item-buffer"; break;
+ case ItemType::GetImageData: ts << "get-image-data"; break;
case ItemType::PutImageData: ts << "put-image-data"; break;
case ItemType::PaintFrameForMedia: ts << "paint-frame-for-media"; break;
case ItemType::StrokeRect: ts << "stroke-rect"; break;
@@ -1225,6 +1227,9 @@
case ItemType::MetaCommandChangeItemBuffer:
ts << item.get<MetaCommandChangeItemBuffer>();
break;
+ case ItemType::GetImageData:
+ ts << item.get<GetImageData>();
+ break;
case ItemType::PutImageData:
ts << item.get<PutImageData>();
break;
Modified: trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItems.h (274837 => 274838)
--- trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItems.h 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebCore/platform/graphics/displaylists/DisplayListItems.h 2021-03-23 01:53:56 UTC (rev 274838)
@@ -51,6 +51,15 @@
namespace DisplayList {
+/* isInlineItem indicates whether the object needs to be passed through IPC::Encoder in order to serialize,
+ * or whether we can just use placement new and be done.
+ * It needs to match (1) RemoteImageBufferProxy::encodeItem(), (2) RemoteRenderingBackend::decodeItem(),
+ * and (3) isInlineItem() in DisplayListItemType.cpp.
+ *
+ * isDrawingItem indicates if this command can affect dirty rects.
+ * We can do things like skip drawing items when replaying them if their extents don't intersect with the current clip.
+ * It needs to match isDrawingItem() in DisplayListItemType.cpp. */
+
class Save {
public:
static constexpr ItemType itemType = ItemType::Save;
@@ -1885,6 +1894,26 @@
FloatRect m_rect;
};
+class GetImageData {
+public:
+ static constexpr ItemType itemType = ItemType::GetImageData;
+ static constexpr bool isInlineItem = true;
+ static constexpr bool isDrawingItem = false;
+
+ GetImageData(WebCore::AlphaPremultiplication outputFormat, const WebCore::IntRect& srcRect)
+ : m_srcRect(srcRect)
+ , m_outputFormat(outputFormat)
+ {
+ }
+
+ AlphaPremultiplication outputFormat() const { return m_outputFormat; }
+ IntRect srcRect() const { return m_srcRect; }
+
+private:
+ IntRect m_srcRect;
+ AlphaPremultiplication m_outputFormat;
+};
+
class PutImageData {
public:
static constexpr ItemType itemType = ItemType::PutImageData;
@@ -1900,8 +1929,6 @@
IntPoint destPoint() const { return m_destPoint; }
AlphaPremultiplication destFormat() const { return m_destFormat; }
- NO_RETURN_DUE_TO_ASSERT void apply(GraphicsContext&) const;
-
Optional<FloatRect> localBounds(const GraphicsContext&) const { return WTF::nullopt; }
Optional<FloatRect> globalBounds() const { return {{ m_destPoint, m_srcRect.size() }}; }
Modified: trunk/Source/WebCore/platform/graphics/displaylists/DisplayListRecorder.cpp (274837 => 274838)
--- trunk/Source/WebCore/platform/graphics/displaylists/DisplayListRecorder.cpp 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebCore/platform/graphics/displaylists/DisplayListRecorder.cpp 2021-03-23 01:53:56 UTC (rev 274838)
@@ -55,6 +55,11 @@
LOG(DisplayLists, "Recorded display list:\n%s", m_displayList.description().data());
}
+void Recorder::getImageData(AlphaPremultiplication outputFormat, const IntRect& sourceRect)
+{
+ append<GetImageData>(outputFormat, sourceRect);
+}
+
void Recorder::putImageData(WebCore::AlphaPremultiplication inputFormat, const WebCore::ImageData& imageData, const WebCore::IntRect& srcRect, const WebCore::IntPoint& destPoint, WebCore::AlphaPremultiplication destFormat)
{
append<PutImageData>(inputFormat, imageData, srcRect, destPoint, destFormat);
Modified: trunk/Source/WebCore/platform/graphics/displaylists/DisplayListRecorder.h (274837 => 274838)
--- trunk/Source/WebCore/platform/graphics/displaylists/DisplayListRecorder.h 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebCore/platform/graphics/displaylists/DisplayListRecorder.h 2021-03-23 01:53:56 UTC (rev 274838)
@@ -57,6 +57,7 @@
WEBCORE_EXPORT Recorder(GraphicsContext&, DisplayList&, const GraphicsContextState&, const FloatRect& initialClip, const AffineTransform&, Delegate* = nullptr, DrawGlyphsRecorder::DrawGlyphsDeconstruction = DrawGlyphsRecorder::DrawGlyphsDeconstruction::Deconstruct);
WEBCORE_EXPORT virtual ~Recorder();
+ WEBCORE_EXPORT void getImageData(AlphaPremultiplication outputFormat, const IntRect& sourceRect);
WEBCORE_EXPORT void putImageData(AlphaPremultiplication inputFormat, const ImageData&, const IntRect& srcRect, const IntPoint& destPoint, AlphaPremultiplication destFormat);
bool isEmpty() const { return m_displayList.isEmpty(); }
Modified: trunk/Source/WebKit/ChangeLog (274837 => 274838)
--- trunk/Source/WebKit/ChangeLog 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebKit/ChangeLog 2021-03-23 01:53:56 UTC (rev 274838)
@@ -1,3 +1,15 @@
+2021-03-22 Myles C. Maxfield <[email protected]>
+
+ [GPU Process]: Improve getImageData() perf part 1: Add a GetImageData display list item
+ https://bugs.webkit.org/show_bug.cgi?id=223544
+
+ Reviewed by Wenson Hsieh.
+
+ * GPUProcess/graphics/RemoteImageBuffer.h:
+ * GPUProcess/graphics/RemoteRenderingBackend.cpp:
+ (WebKit::RemoteRenderingBackend::decodeItem):
+ * WebProcess/GPU/graphics/RemoteImageBufferProxy.h:
+
2021-03-22 Jean-Yves Avenard <[email protected]>
Move management of RemoteCommandListener from MediaSessionManagerCocoa into NowPlayingManager
Modified: trunk/Source/WebKit/GPUProcess/graphics/RemoteImageBuffer.h (274837 => 274838)
--- trunk/Source/WebKit/GPUProcess/graphics/RemoteImageBuffer.h 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebKit/GPUProcess/graphics/RemoteImageBuffer.h 2021-03-23 01:53:56 UTC (rev 274838)
@@ -76,6 +76,13 @@
private:
bool apply(WebCore::DisplayList::ItemHandle item, WebCore::GraphicsContext& context) override
{
+ if (item.is<WebCore::DisplayList::GetImageData>()) {
+ auto& getImageDataItem = item.get<WebCore::DisplayList::GetImageData>();
+ UNUSED_VARIABLE(getImageDataItem);
+ // FIXME: https://bugs.webkit.org/show_bug.cgi?id=220649 Implement this.
+ return true;
+ }
+
if (item.is<WebCore::DisplayList::PutImageData>()) {
auto& putImageDataItem = item.get<WebCore::DisplayList::PutImageData>();
putImageData(putImageDataItem.inputFormat(), putImageDataItem.imageData(), putImageDataItem.srcRect(), putImageDataItem.destPoint(), putImageDataItem.destFormat());
Modified: trunk/Source/WebKit/GPUProcess/graphics/RemoteRenderingBackend.cpp (274837 => 274838)
--- trunk/Source/WebKit/GPUProcess/graphics/RemoteRenderingBackend.cpp 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebKit/GPUProcess/graphics/RemoteRenderingBackend.cpp 2021-03-23 01:53:56 UTC (rev 274838)
@@ -436,6 +436,10 @@
Optional<DisplayList::ItemHandle> WARN_UNUSED_RETURN RemoteRenderingBackend::decodeItem(const uint8_t* data, size_t length, DisplayList::ItemType type, uint8_t* handleLocation)
{
+ /* This needs to match (1) isInlineItem() in DisplayListItemType.cpp, (2) RemoteImageBufferProxy::encodeItem(),
+ * and (3) all the "static constexpr bool isInlineItem"s inside the individual item classes.
+ * See the comment at the top of DisplayListItems.h for why. */
+
switch (type) {
case DisplayList::ItemType::ClipOutToPath:
return decodeAndCreate<DisplayList::ClipOutToPath>(data, length, handleLocation);
@@ -520,7 +524,8 @@
#endif
case DisplayList::ItemType::StrokeRect:
case DisplayList::ItemType::StrokeLine:
- case DisplayList::ItemType::Translate: {
+ case DisplayList::ItemType::Translate:
+ case DisplayList::ItemType::GetImageData: {
ASSERT_NOT_REACHED();
break;
}
Modified: trunk/Source/WebKit/WebProcess/GPU/graphics/RemoteImageBufferProxy.h (274837 => 274838)
--- trunk/Source/WebKit/WebProcess/GPU/graphics/RemoteImageBufferProxy.h 2021-03-23 01:11:46 UTC (rev 274837)
+++ trunk/Source/WebKit/WebProcess/GPU/graphics/RemoteImageBufferProxy.h 2021-03-23 01:53:56 UTC (rev 274838)
@@ -143,15 +143,6 @@
return m_backend.get();
}
- RefPtr<WebCore::ImageData> getImageData(WebCore::AlphaPremultiplication outputFormat, const WebCore::IntRect& srcRect) const override
- {
- if (UNLIKELY(!m_remoteRenderingBackendProxy))
- return nullptr;
-
- const_cast<RemoteImageBufferProxy*>(this)->flushDrawingContext();
- return m_remoteRenderingBackendProxy->getImageData(outputFormat, srcRect, m_renderingResourceIdentifier);
- }
-
String toDataURL(const String& mimeType, Optional<double> quality, WebCore::PreserveResolution preserveResolution) const override
{
if (UNLIKELY(!m_remoteRenderingBackendProxy))
@@ -197,6 +188,19 @@
return bitmap->createImage();
}
+ RefPtr<WebCore::ImageData> getImageData(WebCore::AlphaPremultiplication outputFormat, const WebCore::IntRect& srcRect) const override
+ {
+ if (UNLIKELY(!m_remoteRenderingBackendProxy))
+ return nullptr;
+
+ auto& mutableThis = const_cast<RemoteImageBufferProxy&>(*this);
+ mutableThis.m_drawingContext.recorder().getImageData(outputFormat, srcRect);
+ // FIXME: https://bugs.webkit.org/show_bug.cgi?id=220649 Use the recorded command instead of the synchronous IPC message.
+
+ const_cast<RemoteImageBufferProxy*>(this)->flushDrawingContext();
+ return m_remoteRenderingBackendProxy->getImageData(outputFormat, srcRect, m_renderingResourceIdentifier);
+ }
+
void putImageData(WebCore::AlphaPremultiplication inputFormat, const WebCore::ImageData& imageData, const WebCore::IntRect& srcRect, const WebCore::IntPoint& destPoint = { }, WebCore::AlphaPremultiplication destFormat = WebCore::AlphaPremultiplication::Premultiplied) override
{
// The math inside ImageData::create() doesn't agree with the math inside ImageBufferBackend::putImageData() about how m_resolutionScale interacts with the data in the ImageBuffer.
@@ -299,6 +303,10 @@
RefPtr<WebCore::SharedBuffer> encodeItem(WebCore::DisplayList::ItemHandle item) const override
{
+ /* This needs to match (1) isInlineItem() in DisplayListItemType.cpp, (2) RemoteRenderingBackend::decodeItem(),
+ * and (3) all the "static constexpr bool isInlineItem"s inside the individual item classes.
+ * See the comment at the top of DisplayListItems.h for why. */
+
switch (item.type()) {
case WebCore::DisplayList::ItemType::ClipOutToPath:
return IPC::Encoder::encodeSingleObject<WebCore::DisplayList::ClipOutToPath>(item.get<WebCore::DisplayList::ClipOutToPath>());
