Modified: trunk/Source/WebCore/ChangeLog (288476 => 288477)
--- trunk/Source/WebCore/ChangeLog 2022-01-24 23:20:23 UTC (rev 288476)
+++ trunk/Source/WebCore/ChangeLog 2022-01-24 23:47:36 UTC (rev 288477)
@@ -1,3 +1,42 @@
+2022-01-24 Wenson Hsieh <[email protected]>
+
+ [macOS] Update Pasteboard::read to prioritize native representations over TIFF
+ https://bugs.webkit.org/show_bug.cgi?id=190101
+ rdar://44879244
+
+ Reviewed by Ryosuke Niwa.
+
+ Currently on macOS, when pasting image data from the pasteboard into richly editable elements, we attempt to
+ read the image data by checking a hard-coded list of types; in order, these are: TIFF, PDF, PNG, and finally
+ JPEG. Since AppKit automatically provides TIFF as an available type for compatibility when writing non-TIFF
+ image data to the pasteboard, this means that WebKit automatically transcodes all images to TIFF when pasting
+ images in editable areas.
+
+ This leads to unexpected behavior in Mail compose when using the continuity camera to insert images, since all
+ drawings and photos end up being inserted as (often much larger) TIFF files instead of JPEG and PNG,
+ respectively. To address this, we adjust `Pasteboard::read` to prioritize image formats in the hard-coded list
+ above that were written directly to the pasteboard by the copier (i.e. without transcoding) before reading from
+ the remaining types that were added to the pasteboard by the system (and thus require extra transcoding).
+
+ In the case where only PNG and JPEG data was written to the pasteboard by the copier, this means we'll read
+ images as PNG and JPEG data directly. If TIFF was explicitly written as an image format to the pasteboard, this
+ patch will preserve existing behavior by reading the image data as TIFF.
+
+ Note that this existing logic for reading images directly into web content is an existing privacy risk, since
+ AppKit's pasteboard code preserves all metadata properties when transcoding between image formats. In the
+ future, we should add a step here to strip out as much metadata as possible here, with an allow-list of metadata
+ properties that are essential to preserving image fidelity (e.g. orientation).
+
+ See: <https://webkit.org/b/235534> for more details.
+
+ Test: WKAttachmentTests.PastingPreservesImageFormat
+
+ * platform/mac/PasteboardMac.mm:
+ (WebCore::Pasteboard::read):
+
+ Refactor this logic so that we first try to read any natively written image formats that happen to be one of
+ ("TIFF", "PDF", "PNG", "JPEG"), before trying to read any other types that would otherwise require transcoding.
+
2022-01-24 Eric Carlson <[email protected]>
REGRESSION (iOS 15): HTMLAudioElement fails to load new audio when device is locked or safari is in background
Modified: trunk/Source/WebCore/platform/mac/PasteboardMac.mm (288476 => 288477)
--- trunk/Source/WebCore/platform/mac/PasteboardMac.mm 2022-01-24 23:20:23 UTC (rev 288476)
+++ trunk/Source/WebCore/platform/mac/PasteboardMac.mm 2022-01-24 23:47:36 UTC (rev 288477)
@@ -438,13 +438,27 @@
void Pasteboard::read(PasteboardWebContentReader& reader, WebContentReadingPolicy policy, std::optional<size_t> itemIndex)
{
auto& strategy = *platformStrategies()->pasteboardStrategy();
+ auto platformTypesFromItems = [](const Vector<PasteboardItemInfo>& items) {
+ HashSet<String> types;
+ for (auto& item : items) {
+ for (auto& type : item.platformTypesByFidelity)
+ types.add(type);
+ }
+ return types;
+ };
+ HashSet<String> nonTranscodedTypes;
Vector<String> types;
if (itemIndex) {
- if (auto itemInfo = strategy.informationForItemAtIndex(*itemIndex, m_pasteboardName, m_changeCount, context()))
+ if (auto itemInfo = strategy.informationForItemAtIndex(*itemIndex, m_pasteboardName, m_changeCount, context())) {
types = itemInfo->platformTypesByFidelity;
- } else
+ nonTranscodedTypes = platformTypesFromItems({ *itemInfo });
+ }
+ } else {
strategy.getTypes(types, m_pasteboardName, context());
+ if (auto allItems = strategy.allPasteboardItemInfo(m_pasteboardName, m_changeCount, context()))
+ nonTranscodedTypes = platformTypesFromItems(*allItems);
+ }
reader.contentOrigin = readOrigin();
@@ -519,51 +533,46 @@
if (policy == WebContentReadingPolicy::OnlyRichTextTypes)
return;
- if (types.contains(String(legacyTIFFPasteboardType()))) {
- if (auto buffer = readBufferAtPreferredItemIndex(legacyTIFFPasteboardType(), itemIndex, strategy, m_pasteboardName, context())) {
- if (m_changeCount != changeCount() || reader.readImage(buffer.releaseNonNull(), "image/tiff"_s))
- return;
- }
- }
+ using ImageReadingInfo = std::tuple<String, ASCIILiteral>;
+ ALLOW_DEPRECATED_DECLARATIONS_BEGIN
+ const std::array<ImageReadingInfo, 6> imageTypesToRead { {
+ { String(legacyTIFFPasteboardType()), "image/tiff"_s },
+ { String(NSPasteboardTypeTIFF), "image/tiff"_s },
+ { String(legacyPDFPasteboardType()), "application/pdf"_s },
+ { String(NSPasteboardTypePDF), "application/pdf"_s },
+ { String(kUTTypePNG), "image/png"_s },
+ { String(kUTTypeJPEG), "image/jpeg"_s }
+ } };
+ ALLOW_DEPRECATED_DECLARATIONS_END
- if (types.contains(String(NSPasteboardTypeTIFF))) {
- if (auto buffer = readBufferAtPreferredItemIndex(NSPasteboardTypeTIFF, itemIndex, strategy, m_pasteboardName, context())) {
- if (m_changeCount != changeCount() || reader.readImage(buffer.releaseNonNull(), "image/tiff"_s))
- return;
- }
- }
+ auto tryToReadImage = [&] (const String& pasteboardType, ASCIILiteral mimeType) {
+ if (!types.contains(pasteboardType))
+ return false;
- if (types.contains(String(legacyPDFPasteboardType()))) {
- if (auto buffer = readBufferAtPreferredItemIndex(legacyPDFPasteboardType(), itemIndex, strategy, m_pasteboardName, context())) {
- if (m_changeCount != changeCount() || reader.readImage(buffer.releaseNonNull(), "application/pdf"_s))
- return;
- }
- }
+ auto buffer = readBufferAtPreferredItemIndex(pasteboardType, itemIndex, strategy, m_pasteboardName, context());
+ if (m_changeCount != changeCount())
+ return true;
- if (types.contains(String(NSPasteboardTypePDF))) {
- if (auto buffer = readBufferAtPreferredItemIndex(NSPasteboardTypePDF, itemIndex, strategy, m_pasteboardName, context())) {
- if (m_changeCount != changeCount() || reader.readImage(buffer.releaseNonNull(), "application/pdf"_s))
- return;
- }
- }
+ if (!buffer)
+ return false;
-ALLOW_DEPRECATED_DECLARATIONS_BEGIN
- if (types.contains(String(kUTTypePNG))) {
- if (auto buffer = readBufferAtPreferredItemIndex(kUTTypePNG, itemIndex, strategy, m_pasteboardName, context())) {
- if (m_changeCount != changeCount() || reader.readImage(buffer.releaseNonNull(), "image/png"_s))
- return;
+ return reader.readImage(buffer.releaseNonNull(), mimeType);
+ };
+
+ Vector<ImageReadingInfo, 6> transcodedImageTypesToRead;
+ for (auto& [pasteboardType, mimeType] : imageTypesToRead) {
+ if (!nonTranscodedTypes.contains(pasteboardType)) {
+ transcodedImageTypesToRead.append({ pasteboardType, mimeType });
+ continue;
}
+ if (tryToReadImage(pasteboardType, mimeType))
+ return;
}
-ALLOW_DEPRECATED_DECLARATIONS_END
-ALLOW_DEPRECATED_DECLARATIONS_BEGIN
- if (types.contains(String(kUTTypeJPEG))) {
- if (auto buffer = readBufferAtPreferredItemIndex(kUTTypeJPEG, itemIndex, strategy, m_pasteboardName, context())) {
- if (m_changeCount != changeCount() || reader.readImage(buffer.releaseNonNull(), "image/jpeg"_s))
- return;
- }
+ for (auto& [pasteboardType, mimeType] : transcodedImageTypesToRead) {
+ if (tryToReadImage(pasteboardType, mimeType))
+ return;
}
-ALLOW_DEPRECATED_DECLARATIONS_END
if (types.contains(String(legacyURLPasteboardType()))) {
URL url = "" context());
Modified: trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/WKAttachmentTests.mm (288476 => 288477)
--- trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/WKAttachmentTests.mm 2022-01-24 23:20:23 UTC (rev 288476)
+++ trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/WKAttachmentTests.mm 2022-01-24 23:47:36 UTC (rev 288477)
@@ -260,6 +260,16 @@
return [NSData dataWithContentsOfURL:testPDFFileURL()];
}
+static NSURL *testJPEGFileURL()
+{
+ return [[NSBundle mainBundle] URLForResource:@"sunset-in-cupertino-600px" withExtension:@"jpg" subdirectory:@"TestWebKitAPI.resources"];
+}
+
+static NSData *testJPEGData()
+{
+ return [NSData dataWithContentsOfURL:testJPEGFileURL()];
+}
+
@implementation TestWKWebView (AttachmentTesting)
- (_WKAttachment *)synchronouslyInsertAttachmentWithFileWrapper:(NSFileWrapper *)fileWrapper contentType:(NSString *)contentType
@@ -616,7 +626,37 @@
#endif
}
+void platformCopyPDF()
+{
#if PLATFORM(MAC)
+ NSPasteboard *pasteboard = [NSPasteboard generalPasteboard];
+ [pasteboard declareTypes:@[(__bridge NSString *)kUTTypePDF] owner:nil];
+ [pasteboard setData:testPDFData() forType:(__bridge NSString *)kUTTypePDF];
+#elif PLATFORM(IOS_FAMILY)
+ UIPasteboard *pasteboard = [UIPasteboard generalPasteboard];
+ auto item = adoptNS([[NSItemProvider alloc] init]);
+ [item setPreferredPresentationStyle:UIPreferredPresentationStyleAttachment];
+ [item registerData:testPDFData() type:(__bridge NSString *)kUTTypePDF];
+ pasteboard.itemProviders = @[ item.get() ];
+#endif
+}
+
+void platformCopyJPEG()
+{
+#if PLATFORM(MAC)
+ NSPasteboard *pasteboard = [NSPasteboard generalPasteboard];
+ [pasteboard declareTypes:@[(__bridge NSString *)kUTTypeJPEG] owner:nil];
+ [pasteboard setData:testJPEGData() forType:(__bridge NSString *)kUTTypeJPEG];
+#elif PLATFORM(IOS_FAMILY)
+ UIPasteboard *pasteboard = [UIPasteboard generalPasteboard];
+ auto item = adoptNS([[NSItemProvider alloc] init]);
+ [item setPreferredPresentationStyle:UIPreferredPresentationStyleAttachment];
+ [item registerData:testJPEGData() type:(__bridge NSString *)kUTTypeJPEG];
+ pasteboard.itemProviders = @[ item.get() ];
+#endif
+}
+
+#if PLATFORM(MAC)
typedef NSImage PlatformImage;
#else
typedef UIImage PlatformImage;
@@ -1587,7 +1627,6 @@
TEST(WKAttachmentTests, SetFileWrapperForPDFImageAttachment)
{
auto webView = webViewForTestingAttachments();
- [webView evaluateJavaScript:@"document.body.appendChild()" completionHandler:nil];
NSString *identifier = [webView stringByEvaluatingJavaScript:@"const i = document.createElement('img'); document.body.appendChild(i); HTMLAttachmentElement.getAttachmentIdentifier(i)"];
auto attachment = retainPtr([webView _attachmentForIdentifier:identifier]);
@@ -1602,6 +1641,35 @@
EXPECT_FALSE([attachment isConnected]);
}
+TEST(WKAttachmentTests, PastingPreservesImageFormat)
+{
+ auto webView = webViewForTestingAttachments();
+ {
+ platformCopyPNG();
+ ObserveAttachmentUpdatesForScope observer(webView.get());
+ [webView _synchronouslyExecuteEditCommand:@"Paste" argument:nil];
+ EXPECT_EQ(1U, observer.observer().inserted.count);
+ _WKAttachment *attachment = observer.observer().inserted[0];
+ EXPECT_WK_STREQ("image/png", attachment.info.contentType);
+ }
+ {
+ platformCopyPDF();
+ ObserveAttachmentUpdatesForScope observer(webView.get());
+ [webView _synchronouslyExecuteEditCommand:@"Paste" argument:nil];
+ EXPECT_EQ(1U, observer.observer().inserted.count);
+ _WKAttachment *attachment = observer.observer().inserted[0];
+ EXPECT_WK_STREQ("application/pdf", attachment.info.contentType);
+ }
+ {
+ platformCopyJPEG();
+ ObserveAttachmentUpdatesForScope observer(webView.get());
+ [webView _synchronouslyExecuteEditCommand:@"Paste" argument:nil];
+ EXPECT_EQ(1U, observer.observer().inserted.count);
+ _WKAttachment *attachment = observer.observer().inserted[0];
+ EXPECT_WK_STREQ("image/jpeg", attachment.info.contentType);
+ }
+}
+
#pragma mark - Platform-specific tests
#if PLATFORM(MAC)
