Diff
Modified: trunk/Source/WebCore/ChangeLog (288689 => 288690)
--- trunk/Source/WebCore/ChangeLog 2022-01-27 19:45:03 UTC (rev 288689)
+++ trunk/Source/WebCore/ChangeLog 2022-01-27 19:55:22 UTC (rev 288690)
@@ -1,3 +1,31 @@
+2022-01-27 Wenson Hsieh <[email protected]>
+
+ ImageAnalysisQueue should prioritize elements that intersect the visible viewport
+ https://bugs.webkit.org/show_bug.cgi?id=235664
+ rdar://88131827
+
+ Reviewed by Darin Adler.
+
+ Make few adjustments to the image analysis queueing mechanism, such that we prioritize analyzing images that
+ are in the visible document rect over offscreen images. To do this, we turn the current image element queue into
+ a PriorityQueue instead, which treats images that are in the visible viewport (at enqueueing time) as higher
+ priority over images that are not in the visible viewport, and otherwise treats images that were queued earlier
+ as higher priority in the case where both images are either visible or not visible.
+
+ Test: ImageAnalysisTests.ImageAnalysisPrioritizesVisibleImages
+
+ * page/ImageAnalysisQueue.cpp:
+ (WebCore::ImageAnalysisQueue::enqueueIfNeeded):
+ (WebCore::ImageAnalysisQueue::resumeProcessingSoon):
+ (WebCore::ImageAnalysisQueue::resumeProcessing):
+ (WebCore::ImageAnalysisQueue::clear):
+ * page/ImageAnalysisQueue.h:
+
+ Swap out the Deque for a PriorityQueue, and introduce ImageAnalysisQueue::Task to represent an image analysis
+ task (which can be compared against other tasks to determine priority using `firstIsHigherPriority()` below).
+
+ (WebCore::ImageAnalysisQueue::firstIsHigherPriority):
+
2022-01-27 Gavin Phillips <[email protected]>
Detect failed ThreadableLoader creation in FileReaderLoader.
Modified: trunk/Source/WebCore/page/ImageAnalysisQueue.cpp (288689 => 288690)
--- trunk/Source/WebCore/page/ImageAnalysisQueue.cpp 2022-01-27 19:45:03 UTC (rev 288689)
+++ trunk/Source/WebCore/page/ImageAnalysisQueue.cpp 2022-01-27 19:55:22 UTC (rev 288690)
@@ -30,10 +30,12 @@
#include "Chrome.h"
#include "ChromeClient.h"
+#include "FrameView.h"
#include "HTMLCollection.h"
#include "HTMLImageElement.h"
#include "ImageOverlay.h"
#include "RenderImage.h"
+#include "RenderView.h"
#include "Timer.h"
namespace WebCore {
@@ -67,10 +69,23 @@
if (!m_queuedElements.add(element).isNewEntry)
return;
- m_queue.append({ element });
- resumeProcessing();
+ Ref view = renderer.view().frameView();
+ m_queue.enqueue({
+ element,
+ renderer.isVisibleInDocumentRect(view->windowToContents(view->windowClipRect())) ? Priority::High : Priority::Low,
+ nextTaskNumber()
+ });
+ resumeProcessingSoon();
}
+void ImageAnalysisQueue::resumeProcessingSoon()
+{
+ if (m_queue.isEmpty() || m_resumeProcessingTimer.isActive())
+ return;
+
+ m_resumeProcessingTimer.startOneShot(resumeProcessingDelay);
+}
+
void ImageAnalysisQueue::enqueueAllImages(Document& document, const String& identifier)
{
if (!m_page)
@@ -91,8 +106,7 @@
return;
while (!m_queue.isEmpty() && m_pendingRequestCount < maximumPendingImageAnalysisCount) {
- auto weakElement = m_queue.takeFirst();
- RefPtr element = weakElement.get();
+ RefPtr element = m_queue.dequeue().element.get();
if (!element || !element->isConnected())
continue;
@@ -99,14 +113,13 @@
m_pendingRequestCount++;
m_page->resetTextRecognitionResult(*element);
m_page->chrome().client().requestTextRecognition(*element, m_identifier, [this, page = m_page] (auto&&) {
- if (!page)
+ if (!page || page->imageAnalysisQueueIfExists() != this)
return;
if (m_pendingRequestCount)
m_pendingRequestCount--;
- if (!m_queue.isEmpty() && !m_resumeProcessingTimer.isActive())
- m_resumeProcessingTimer.startOneShot(resumeProcessingDelay);
+ resumeProcessingSoon();
});
}
}
@@ -116,9 +129,10 @@
// FIXME: This should cancel pending requests in addition to emptying the task queue.
m_pendingRequestCount = 0;
m_resumeProcessingTimer.stop();
- m_queue.clear();
+ m_queue = { };
m_queuedElements.clear();
m_identifier = { };
+ m_currentTaskNumber = 0;
}
} // namespace WebCore
Modified: trunk/Source/WebCore/page/ImageAnalysisQueue.h (288689 => 288690)
--- trunk/Source/WebCore/page/ImageAnalysisQueue.h 2022-01-27 19:45:03 UTC (rev 288689)
+++ trunk/Source/WebCore/page/ImageAnalysisQueue.h 2022-01-27 19:55:22 UTC (rev 288690)
@@ -27,8 +27,8 @@
#if ENABLE(IMAGE_ANALYSIS)
-#include <wtf/Deque.h>
#include <wtf/FastMalloc.h>
+#include <wtf/PriorityQueue.h>
#include <wtf/WeakHashSet.h>
#include <wtf/WeakPtr.h>
@@ -51,16 +51,36 @@
void enqueueIfNeeded(HTMLImageElement&);
private:
+ void resumeProcessingSoon();
void resumeProcessing();
+ enum class Priority : bool { Low, High };
+ struct Task {
+ WeakPtr<HTMLImageElement> element;
+ Priority priority { Priority::Low };
+ unsigned taskNumber { 0 };
+ };
+
+ static bool firstIsHigherPriority(const Task&, const Task&);
+ unsigned nextTaskNumber() { return ++m_currentTaskNumber; }
+
String m_identifier;
WeakPtr<Page> m_page;
Timer m_resumeProcessingTimer;
WeakHashSet<HTMLImageElement> m_queuedElements;
- Deque<WeakPtr<HTMLImageElement>> m_queue;
+ PriorityQueue<Task, firstIsHigherPriority> m_queue;
unsigned m_pendingRequestCount { 0 };
+ unsigned m_currentTaskNumber { 0 };
};
+inline bool ImageAnalysisQueue::firstIsHigherPriority(const Task& first, const Task& second)
+{
+ if (first.priority != second.priority)
+ return first.priority == Priority::High;
+
+ return first.taskNumber < second.taskNumber;
+}
+
} // namespace WebCore
#endif // ENABLE(IMAGE_ANALYSIS)
Modified: trunk/Tools/ChangeLog (288689 => 288690)
--- trunk/Tools/ChangeLog 2022-01-27 19:45:03 UTC (rev 288689)
+++ trunk/Tools/ChangeLog 2022-01-27 19:55:22 UTC (rev 288690)
@@ -1,3 +1,38 @@
+2022-01-27 Wenson Hsieh <[email protected]>
+
+ ImageAnalysisQueue should prioritize elements that intersect the visible viewport
+ https://bugs.webkit.org/show_bug.cgi?id=235664
+ rdar://88131827
+
+ Reviewed by Darin Adler.
+
+ Add an API test to verify that batch image analysis prioritizes a visible image over another image that appears
+ first in DOM traversal order, but is offscreen.
+
+ * TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj:
+ * TestWebKitAPI/Tests/WebKitCocoa/ImageAnalysisTests.mm:
+ (TestWebKitAPI::processedRequests):
+ (TestWebKitAPI::processRequestWithResults):
+ (TestWebKitAPI::makeFakeRequest):
+ (TestWebKitAPI::makeImageAnalysisRequestSwizzler):
+ (TestWebKitAPI::processRequestWithError):
+
+ Additionally add a mechanism for swizzling out VKImageAnalyzerRequest initialization to return a mock object
+ (TestVKImageAnalyzerRequest) instead, which can then be used to inspect the CGImage that would've been sent for
+ analysis. See changes in `ImageAnalysisTestingUtilities.*` below for more information.
+
+ (TestWebKitAPI::TEST):
+ * TestWebKitAPI/Tests/WebKitCocoa/offscreen-image.html: Added.
+ * TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.h:
+ * TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.mm:
+ (-[TestVKImageAnalyzerRequest initWithCGImage:orientation:requestType:]):
+ (-[TestVKImageAnalyzerRequest image]):
+ (-[TestVKImageAnalyzerRequest imageURL]):
+ (-[TestVKImageAnalyzerRequest setImageURL:]):
+ (-[TestVKImageAnalyzerRequest pageURL]):
+ (-[TestVKImageAnalyzerRequest setPageURL:]):
+ (TestWebKitAPI::createRequest):
+
2022-01-26 Jonathan Bedard <[email protected]>
[webkitbugspy] JSON encode trackers
Modified: trunk/Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj (288689 => 288690)
--- trunk/Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj 2022-01-27 19:45:03 UTC (rev 288689)
+++ trunk/Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj 2022-01-27 19:55:22 UTC (rev 288690)
@@ -1058,6 +1058,7 @@
F45E15732112CE2900307E82 /* KeyboardInputTestsIOS.mm in Sources */ = {isa = PBXBuildFile; fileRef = F45E15722112CE2900307E82 /* KeyboardInputTestsIOS.mm */; };
F45E15762112CE6200307E82 /* TestInputDelegate.mm in Sources */ = {isa = PBXBuildFile; fileRef = F45E15752112CE6200307E82 /* TestInputDelegate.mm */; };
F45EB60427739F34003571AE /* TestModalContainerControls.mlmodelc in Copy Resources */ = {isa = PBXBuildFile; fileRef = F45EB60327739F2B003571AE /* TestModalContainerControls.mlmodelc */; };
+ F460BEEA27A1F416007B87D9 /* offscreen-image.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = F460BEE727A1F2D6007B87D9 /* offscreen-image.html */; };
F460F657261116EA0064F2B6 /* InjectedBundleHitTest.mm in Sources */ = {isa = PBXBuildFile; fileRef = F460F656261116EA0064F2B6 /* InjectedBundleHitTest.mm */; };
F460F65B261119580064F2B6 /* InjectedBundleHitTestPlugIn.mm in Sources */ = {isa = PBXBuildFile; fileRef = F460F659261117E70064F2B6 /* InjectedBundleHitTestPlugIn.mm */; };
F460F669261263370064F2B6 /* simple-responsive-page.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = F460F668261262C70064F2B6 /* simple-responsive-page.html */; };
@@ -1499,6 +1500,7 @@
466C3843210637DE006A88DE /* notify-resourceLoadObserver.html in Copy Resources */,
CDB5DFFF213610FA00D3E189 /* now-playing.html in Copy Resources */,
93E2D2761ED7D53200FA76F6 /* offscreen-iframe-of-media-document.html in Copy Resources */,
+ F460BEEA27A1F416007B87D9 /* offscreen-image.html in Copy Resources */,
074994421EA5034B000DA44F /* ondevicechange.html in Copy Resources */,
1DAA52CC243BE805001A3159 /* one-video.html in Copy Resources */,
CEA6CF2819CCF69D0064F5A7 /* open-and-close-window.html in Copy Resources */,
@@ -3035,6 +3037,7 @@
F45E15752112CE6200307E82 /* TestInputDelegate.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = TestInputDelegate.mm; sourceTree = "<group>"; };
F45EB60327739F2B003571AE /* TestModalContainerControls.mlmodelc */ = {isa = PBXFileReference; lastKnownFileType = wrapper; path = TestModalContainerControls.mlmodelc; sourceTree = "<group>"; };
F45EB6062773C371003571AE /* ModalContainerObservation.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = ModalContainerObservation.mm; sourceTree = "<group>"; };
+ F460BEE727A1F2D6007B87D9 /* offscreen-image.html */ = {isa = PBXFileReference; lastKnownFileType = text.html; path = "offscreen-image.html"; sourceTree = "<group>"; };
F460F656261116EA0064F2B6 /* InjectedBundleHitTest.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = InjectedBundleHitTest.mm; sourceTree = "<group>"; };
F460F659261117E70064F2B6 /* InjectedBundleHitTestPlugIn.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = InjectedBundleHitTestPlugIn.mm; sourceTree = "<group>"; };
F460F65A2611183F0064F2B6 /* InjectedBundleHitTestProtocol.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = InjectedBundleHitTestProtocol.h; sourceTree = "<group>"; };
@@ -4186,6 +4189,7 @@
466C3842210637CE006A88DE /* notify-resourceLoadObserver.html */,
CDB5DFFE21360ED800D3E189 /* now-playing.html */,
93E2D2751ED7D51700FA76F6 /* offscreen-iframe-of-media-document.html */,
+ F460BEE727A1F2D6007B87D9 /* offscreen-image.html */,
7CCB99221D3B44E7003922F6 /* open-multiple-external-url.html */,
931C281B22BC5583001D98C4 /* opendatabase-always-exists.html */,
F49992C5248DABE400034167 /* overflow-hidden.html */,
Modified: trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/ImageAnalysisTests.mm (288689 => 288690)
--- trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/ImageAnalysisTests.mm 2022-01-27 19:45:03 UTC (rev 288689)
+++ trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/ImageAnalysisTests.mm 2022-01-27 19:55:22 UTC (rev 288690)
@@ -53,6 +53,10 @@
#endif // PLATFORM(IOS_FAMILY)
+@interface VKImageAnalyzerRequest (TestSupport)
+@property (nonatomic, readonly) CGImageRef image;
+@end
+
@interface TestWKWebView (ImageAnalysisTests)
- (void)waitForImageAnalysisRequests:(unsigned)numberOfRequests;
#if PLATFORM(IOS_FAMILY)
@@ -92,6 +96,12 @@
namespace TestWebKitAPI {
+static Vector<RetainPtr<VKImageAnalyzerRequest>>& processedRequests()
+{
+ static NeverDestroyed requests = Vector<RetainPtr<VKImageAnalyzerRequest>> { };
+ return requests.get();
+}
+
static RetainPtr<TestWKWebView> createWebViewWithTextRecognitionEnhancements()
{
RetainPtr configuration = [WKWebViewConfiguration _test_configurationWithTestPlugInClassName:@"WebProcessPlugInWithInternals" configureJSCForTesting:YES];
@@ -104,23 +114,33 @@
return adoptNS([[TestWKWebView alloc] initWithFrame:NSMakeRect(0, 0, 300, 300) configuration:configuration.get()]);
}
-static void processRequestWithResults(id, SEL, VKImageAnalyzerRequest *, void (^)(double progress), void (^completion)(VKImageAnalysis *, NSError *))
+static void processRequestWithResults(id, SEL, VKImageAnalyzerRequest *request, void (^)(double progress), void (^completion)(VKImageAnalysis *, NSError *))
{
gDidProcessRequestCount++;
+ processedRequests().append({ request });
completion(createImageAnalysisWithSimpleFixedResults().get(), nil);
}
+static VKImageAnalyzerRequest *makeFakeRequest(id, SEL, CGImageRef image, VKImageOrientation orientation, VKAnalysisTypes requestTypes)
+{
+ return createRequest(image, orientation, requestTypes).leakRef();
+}
+
template <typename FunctionType>
-InstanceMethodSwizzler makeImageAnalysisRequestSwizzler(FunctionType function)
+std::pair<std::unique_ptr<InstanceMethodSwizzler>, std::unique_ptr<InstanceMethodSwizzler>> makeImageAnalysisRequestSwizzler(FunctionType function)
{
- return InstanceMethodSwizzler { PAL::getVKImageAnalyzerClass(), @selector(processRequest:progressHandler:completionHandler:), reinterpret_cast<IMP>(function) };
+ return std::pair {
+ makeUnique<InstanceMethodSwizzler>(PAL::getVKImageAnalyzerClass(), @selector(processRequest:progressHandler:completionHandler:), reinterpret_cast<IMP>(function)),
+ makeUnique<InstanceMethodSwizzler>(PAL::getVKImageAnalyzerRequestClass(), @selector(initWithCGImage:orientation:requestType:), reinterpret_cast<IMP>(makeFakeRequest))
+ };
}
#if PLATFORM(IOS_FAMILY)
-static void processRequestWithError(id, SEL, VKImageAnalyzerRequest *, void (^)(double progress), void (^completion)(VKImageAnalysis *analysis, NSError *error))
+static void processRequestWithError(id, SEL, VKImageAnalyzerRequest *request, void (^)(double progress), void (^completion)(VKImageAnalysis *analysis, NSError *error))
{
gDidProcessRequestCount++;
+ processedRequests().append({ request });
completion(nil, [NSError errorWithDomain:NSCocoaErrorDomain code:1 userInfo:nil]);
}
@@ -224,6 +244,22 @@
EXPECT_EQ(gDidProcessRequestCount, 5U);
}
+TEST(ImageAnalysisTests, ImageAnalysisPrioritizesVisibleImages)
+{
+ auto requestSwizzler = makeImageAnalysisRequestSwizzler(processRequestWithResults);
+ auto webView = createWebViewWithTextRecognitionEnhancements();
+ [webView synchronouslyLoadTestPageNamed:@"offscreen-image"];
+ [webView _startImageAnalysis:nil];
+ [webView waitForImageAnalysisRequests:2];
+
+ auto firstRequestedImage = [processedRequests().first() image];
+ auto lastRequestedImage = [processedRequests().last() image];
+ EXPECT_EQ(200U, CGImageGetWidth(firstRequestedImage));
+ EXPECT_EQ(150U, CGImageGetHeight(firstRequestedImage));
+ EXPECT_EQ(600U, CGImageGetWidth(lastRequestedImage));
+ EXPECT_EQ(450U, CGImageGetHeight(lastRequestedImage));
+}
+
} // namespace TestWebKitAPI
#endif // ENABLE(IMAGE_ANALYSIS)
Added: trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/offscreen-image.html (0 => 288690)
--- trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/offscreen-image.html (rev 0)
+++ trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/offscreen-image.html 2022-01-27 19:55:22 UTC (rev 288690)
@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta name="viewport" content="initial-scale=1">
+<style>
+img {
+ position: absolute;
+}
+</style>
+</head>
+<body>
+<img style="top: 9000px;" src=""
+<img src=""
+</body>
+</html>
Modified: trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.h (288689 => 288690)
--- trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.h 2022-01-27 19:45:03 UTC (rev 288689)
+++ trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.h 2022-01-27 19:55:22 UTC (rev 288690)
@@ -25,6 +25,7 @@
#pragma once
+#import <pal/spi/cocoa/VisionKitCoreSPI.h>
#import <wtf/RetainPtr.h>
#if HAVE(VK_IMAGE_ANALYSIS)
@@ -42,6 +43,7 @@
RetainPtr<VKWKTextInfo> createTextInfo(NSString *text, VKQuad *);
RetainPtr<VKWKLineInfo> createLineInfo(NSString *text, VKQuad *, NSArray<VKWKTextInfo *> *);
RetainPtr<VKImageAnalysis> createImageAnalysis(NSArray<VKWKLineInfo *> *);
+RetainPtr<VKImageAnalyzerRequest> createRequest(CGImageRef, VKImageOrientation, VKAnalysisTypes);
} // namespace TestWebKitAPI
Modified: trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.mm (288689 => 288690)
--- trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.mm 2022-01-27 19:45:03 UTC (rev 288689)
+++ trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.mm 2022-01-27 19:55:22 UTC (rev 288690)
@@ -142,6 +142,60 @@
@end
+@interface TestVKImageAnalyzerRequest : NSObject
+@property (nonatomic, readonly) CGImageRef image;
+@property (nonatomic, readonly) VKImageOrientation orientation;
+@property (nonatomic, readonly) VKAnalysisTypes requestType;
+@property (nonatomic, copy) NSURL *imageURL;
+@property (nonatomic, copy) NSURL *pageURL;
+@end
+
+@implementation TestVKImageAnalyzerRequest {
+ RetainPtr<CGImageRef> _image;
+ VKImageOrientation _orientation;
+ VKAnalysisTypes _requestType;
+ RetainPtr<NSURL> _imageURL;
+ RetainPtr<NSURL> _pageURL;
+}
+
+- (instancetype)initWithCGImage:(CGImageRef)image orientation:(VKImageOrientation)orientation requestType:(VKAnalysisTypes)requestType
+{
+ if (!(self = [super init]))
+ return nil;
+
+ _image = image;
+ _orientation = orientation;
+ _requestType = requestType;
+ return self;
+}
+
+- (CGImageRef)image
+{
+ return _image.get();
+}
+
+- (NSURL *)imageURL
+{
+ return _imageURL.get();
+}
+
+- (void)setImageURL:(NSURL *)url
+{
+ _imageURL = adoptNS(url.copy);
+}
+
+- (NSURL *)pageURL
+{
+ return _pageURL.get();
+}
+
+- (void)setPageURL:(NSURL *)url
+{
+ _pageURL = adoptNS(url.copy);
+}
+
+@end
+
namespace TestWebKitAPI {
RetainPtr<VKQuad> createQuad(CGPoint topLeft, CGPoint topRight, CGPoint bottomLeft, CGPoint bottomRight)
@@ -172,6 +226,11 @@
return createImageAnalysis(@[ line.get() ]);
}
+RetainPtr<VKImageAnalyzerRequest> createRequest(CGImageRef image, VKImageOrientation orientation, VKAnalysisTypes types)
+{
+ return adoptNS(static_cast<VKImageAnalyzerRequest *>([[TestVKImageAnalyzerRequest alloc] initWithCGImage:image orientation:orientation requestType:types]));
+}
+
} // namespace TestWebKitAPI
#endif // HAVE(VK_IMAGE_ANALYSIS)
