- Revision
- 290620
- Author
- wenson_hs...@apple.com
- Date
- 2022-02-28 16:14:24 -0800 (Mon, 28 Feb 2022)
Log Message
Add API testing infrastructure for "Markup Image" and "Copy Cropped Image"
https://bugs.webkit.org/show_bug.cgi?id=236519
rdar://89154213
Reviewed by Tim Horton.
Add API testing support (as well as a new API test) for the "Markup Image" flow.
* TestWebKitAPI/Tests/WebKitCocoa/ImageAnalysisTests.mm:
(TestWebKitAPI::TEST):
(TestWebKitAPI::invokeImageMarkupAction):
* TestWebKitAPI/Tests/WebKitCocoa/WKContentViewEditingActions.mm:
Add an API test that triggers the "Markup Image" action, and verifies the following:
- The first image is replaced with the canned image result (a 215-px-wide image of the WebKit logo).
- The selection range is effectively unchanged.
* TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.h:
* TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.mm:
Add a declaration for ImageAnalysisMarkupSwizzler, a helper class that is used in the new API test to make
VisionKit return a canned image and cropped rect when invoking the SPI that corresponds to "Markup Image"; see
WebKitAdditions change for more information.
* TestWebKitAPI/cocoa/TestWKWebView.h:
* TestWebKitAPI/cocoa/TestWKWebView.mm:
(-[WKWebView selectedText]):
* TestWebKitAPI/ios/TestUIMenuBuilder.h:
* TestWebKitAPI/ios/TestUIMenuBuilder.mm:
(-[TestUIMenuBuilder actionWithTitle:]):
Refactor and rename this to return the found UIAction instead of just a BOOL indicating whether or not it could
be found, and change existing call sites above to use `EXPECT_NULL` and `EXPECT_NOT_NULL`.
(-[TestUIMenuBuilder containsActionWithTitle:]): Deleted.
* TestWebKitAPI/ios/UIKitSPI.h:
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (290619 => 290620)
--- trunk/Tools/ChangeLog 2022-02-28 23:48:00 UTC (rev 290619)
+++ trunk/Tools/ChangeLog 2022-03-01 00:14:24 UTC (rev 290620)
@@ -1,3 +1,42 @@
+2022-02-28 Wenson Hsieh <wenson_hs...@apple.com>
+
+ Add API testing infrastructure for "Markup Image" and "Copy Cropped Image"
+ https://bugs.webkit.org/show_bug.cgi?id=236519
+ rdar://89154213
+
+ Reviewed by Tim Horton.
+
+ Add API testing support (as well as a new API test) for the "Markup Image" flow.
+
+ * TestWebKitAPI/Tests/WebKitCocoa/ImageAnalysisTests.mm:
+ (TestWebKitAPI::TEST):
+ (TestWebKitAPI::invokeImageMarkupAction):
+ * TestWebKitAPI/Tests/WebKitCocoa/WKContentViewEditingActions.mm:
+
+ Add an API test that triggers the "Markup Image" action, and verifies the following:
+ - The first image is replaced with the canned image result (a 215-px-wide image of the WebKit logo).
+ - The selection range is effectively unchanged.
+
+ * TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.h:
+ * TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.mm:
+
+ Add a declaration for ImageAnalysisMarkupSwizzler, a helper class that is used in the new API test to make
+ VisionKit return a canned image and cropped rect when invoking the SPI that corresponds to "Markup Image"; see
+ WebKitAdditions change for more information.
+
+ * TestWebKitAPI/cocoa/TestWKWebView.h:
+ * TestWebKitAPI/cocoa/TestWKWebView.mm:
+ (-[WKWebView selectedText]):
+ * TestWebKitAPI/ios/TestUIMenuBuilder.h:
+ * TestWebKitAPI/ios/TestUIMenuBuilder.mm:
+ (-[TestUIMenuBuilder actionWithTitle:]):
+
+ Refactor and rename this to return the found UIAction instead of just a BOOL indicating whether or not it could
+ be found, and change existing call sites above to use `EXPECT_NULL` and `EXPECT_NOT_NULL`.
+
+ (-[TestUIMenuBuilder containsActionWithTitle:]): Deleted.
+ * TestWebKitAPI/ios/UIKitSPI.h:
+
2022-02-28 Christopher Reid <chris.r...@sony.com>
[Win] vswhere should search for Visual Studio Build tools
Modified: trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/ImageAnalysisTests.mm (290619 => 290620)
--- trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/ImageAnalysisTests.mm 2022-02-28 23:48:00 UTC (rev 290619)
+++ trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/ImageAnalysisTests.mm 2022-03-01 00:14:24 UTC (rev 290620)
@@ -33,6 +33,7 @@
#import "TestInputDelegate.h"
#import "TestUIMenuBuilder.h"
#import "TestWKWebView.h"
+#import "UIKitSPI.h"
#import "WKWebViewConfigurationExtras.h"
#import <WebCore/LocalizedStrings.h>
#import <WebKit/WKWebViewPrivate.h>
@@ -280,7 +281,7 @@
auto menuBuilder = adoptNS([[TestUIMenuBuilder alloc] init]);
[webView buildMenuWithBuilder:menuBuilder.get()];
- EXPECT_TRUE([menuBuilder containsActionWithTitle:WebCore::contextMenuItemTitleMarkupImage()]);
+ EXPECT_NOT_NULL([menuBuilder actionWithTitle:WebCore::contextMenuItemTitleMarkupImage()]);
[webView selectAll:nil];
[webView waitForNextPresentationUpdate];
@@ -287,7 +288,7 @@
[menuBuilder reset];
[webView buildMenuWithBuilder:menuBuilder.get()];
- EXPECT_FALSE([menuBuilder containsActionWithTitle:WebCore::contextMenuItemTitleMarkupImage()]);
+ EXPECT_NULL([menuBuilder actionWithTitle:WebCore::contextMenuItemTitleMarkupImage()]);
[webView objectByEvaluatingJavaScript:@"getSelection().setBaseAndExtent(document.body, 0, document.images[0], 1);"];
[webView waitForNextPresentationUpdate];
@@ -294,9 +295,51 @@
[menuBuilder reset];
[webView buildMenuWithBuilder:menuBuilder.get()];
- EXPECT_TRUE([menuBuilder containsActionWithTitle:WebCore::contextMenuItemTitleMarkupImage()]);
+ EXPECT_NOT_NULL([menuBuilder actionWithTitle:WebCore::contextMenuItemTitleMarkupImage()]);
}
+static void invokeImageMarkupAction(TestWKWebView *webView)
+{
+ // Simulate callout bar appearance.
+ __block bool done = false;
+ [webView.textInputContentView requestRectsToEvadeForSelectionCommandsWithCompletionHandler:^(NSArray<NSValue *> *) {
+ done = true;
+ }];
+ Util::run(&done);
+
+ auto menuBuilder = adoptNS([[TestUIMenuBuilder alloc] init]);
+ [webView buildMenuWithBuilder:menuBuilder.get()];
+ [[menuBuilder actionWithTitle:WebCore::contextMenuItemTitleMarkupImage()] _performActionWithSender:nil];
+ [webView waitForNextPresentationUpdate];
+}
+
+TEST(ImageAnalysisTests, PerformImageAnalysisMarkup)
+{
+ auto iconPath = [NSBundle.mainBundle pathForResource:@"icon" ofType:@"png" inDirectory:@"TestWebKitAPI.resources"];
+ RetainPtr iconImage = [UIImage imageWithContentsOfFile:iconPath];
+ ImageAnalysisMarkupSwizzler swizzler { [iconImage CGImage], CGRectMake(10, 10, 215, 174) };
+
+ auto webView = createWebViewWithTextRecognitionEnhancements();
+ auto inputDelegate = adoptNS([TestInputDelegate new]);
+ [inputDelegate setFocusStartsInputSessionPolicyHandler:[](WKWebView *, id <_WKFocusedElementInfo>) {
+ return _WKFocusStartsInputSessionPolicyAllow;
+ }];
+
+ [webView _setEditable:YES];
+ [webView _setInputDelegate:inputDelegate.get()];
+ [webView synchronouslyLoadTestPageNamed:@"multiple-images"];
+ [webView objectByEvaluatingJavaScript:@"getSelection().setBaseAndExtent(document.body, 0, document.images[0], 1)"];
+ [webView waitForNextPresentationUpdate];
+
+ NSString *previousSelectedText = [webView selectedText];
+ invokeImageMarkupAction(webView.get());
+
+ EXPECT_WK_STREQ(previousSelectedText, [webView selectedText]);
+ Util::waitForConditionWithLogging([&] {
+ return [[webView objectByEvaluatingJavaScript:@"document.images[0].getBoundingClientRect().width"] intValue] == 215;
+ }, 3, @"Expected bounding client rect to become 215.");
+}
+
#endif // ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS) && PLATFORM(IOS_FAMILY)
} // namespace TestWebKitAPI
Modified: trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/WKContentViewEditingActions.mm (290619 => 290620)
--- trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/WKContentViewEditingActions.mm 2022-02-28 23:48:00 UTC (rev 290619)
+++ trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/WKContentViewEditingActions.mm 2022-03-01 00:14:24 UTC (rev 290620)
@@ -93,8 +93,8 @@
auto menuBuilder = adoptNS([[TestUIMenuBuilder alloc] init]);
[webView buildMenuWithBuilder:menuBuilder.get()];
- EXPECT_FALSE([menuBuilder containsActionWithTitle:WebCore::contextMenuItemTagAddHighlightToNewQuickNote()]);
- EXPECT_FALSE([menuBuilder containsActionWithTitle:WebCore::contextMenuItemTagAddHighlightToCurrentQuickNote()]);
+ EXPECT_NULL([menuBuilder actionWithTitle:WebCore::contextMenuItemTagAddHighlightToNewQuickNote()]);
+ EXPECT_NULL([menuBuilder actionWithTitle:WebCore::contextMenuItemTagAddHighlightToCurrentQuickNote()]);
[webView synchronouslyLoadTestPageNamed:@"simple"];
[webView selectAll:nil];
@@ -102,8 +102,8 @@
[menuBuilder reset];
[webView buildMenuWithBuilder:menuBuilder.get()];
- EXPECT_TRUE([menuBuilder containsActionWithTitle:WebCore::contextMenuItemTagAddHighlightToNewQuickNote()]);
- EXPECT_FALSE([menuBuilder containsActionWithTitle:WebCore::contextMenuItemTagAddHighlightToCurrentQuickNote()]);
+ EXPECT_NOT_NULL([menuBuilder actionWithTitle:WebCore::contextMenuItemTagAddHighlightToNewQuickNote()]);
+ EXPECT_NULL([menuBuilder actionWithTitle:WebCore::contextMenuItemTagAddHighlightToCurrentQuickNote()]);
}
#endif // ENABLE(APP_HIGHLIGHTS)
Modified: trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.h (290619 => 290620)
--- trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.h 2022-02-28 23:48:00 UTC (rev 290619)
+++ trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.h 2022-03-01 00:14:24 UTC (rev 290620)
@@ -25,11 +25,12 @@
#pragma once
+#if HAVE(VK_IMAGE_ANALYSIS)
+
+#import "InstanceMethodSwizzler.h"
#import <pal/spi/cocoa/VisionKitCoreSPI.h>
#import <wtf/RetainPtr.h>
-#if HAVE(VK_IMAGE_ANALYSIS)
-
@class VKImageAnalysis;
@class VKQuad;
@class VKWKTextInfo;
@@ -45,6 +46,19 @@
RetainPtr<VKImageAnalysis> createImageAnalysis(NSArray<VKWKLineInfo *> *);
RetainPtr<VKImageAnalyzerRequest> createRequest(CGImageRef, VKImageOrientation, VKAnalysisTypes);
+#if ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+
+class ImageAnalysisMarkupSwizzler {
+public:
+ ImageAnalysisMarkupSwizzler(CGImageRef, CGRect);
+ ~ImageAnalysisMarkupSwizzler() = default;
+
+private:
+ InstanceMethodSwizzler m_imageMarkupRequestSwizzler;
+};
+
+#endif // ENABLE(IMAGE_ANALYSIS_ENHANCEMENTS)
+
} // namespace TestWebKitAPI
#endif // HAVE(VK_IMAGE_ANALYSIS)
Modified: trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.mm (290619 => 290620)
--- trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.mm 2022-02-28 23:48:00 UTC (rev 290619)
+++ trunk/Tools/TestWebKitAPI/cocoa/ImageAnalysisTestingUtilities.mm 2022-03-01 00:14:24 UTC (rev 290620)
@@ -28,7 +28,8 @@
#if HAVE(VK_IMAGE_ANALYSIS)
-#import <pal/spi/cocoa/VisionKitCoreSPI.h>
+#import <wtf/BlockPtr.h>
+#import <pal/cocoa/VisionKitCoreSoftLink.h>
@interface TestVKQuad : NSObject
- (instancetype)initWithTopLeft:(CGPoint)topLeft topRight:(CGPoint)topRight bottomLeft:(CGPoint)bottomLeft bottomRight:(CGPoint)bottomRight;
@@ -233,4 +234,8 @@
} // namespace TestWebKitAPI
+#if USE(APPLE_INTERNAL_SDK)
+#import <WebKitAdditions/ImageAnalysisTestingUtilitiesAdditions.mm>
+#endif
+
#endif // HAVE(VK_IMAGE_ANALYSIS)
Modified: trunk/Tools/TestWebKitAPI/cocoa/TestWKWebView.h (290619 => 290620)
--- trunk/Tools/TestWebKitAPI/cocoa/TestWKWebView.h 2022-02-28 23:48:00 UTC (rev 290619)
+++ trunk/Tools/TestWebKitAPI/cocoa/TestWKWebView.h 2022-03-01 00:14:24 UTC (rev 290620)
@@ -56,6 +56,7 @@
@property (nonatomic, readonly) NSUInteger gpuToWebProcessConnectionCount;
@property (nonatomic, readonly) NSString *contentsAsString;
@property (nonatomic, readonly) NSArray<NSString *> *tagsInBody;
+@property (nonatomic, readonly) NSString *selectedText;
- (void)loadTestPageNamed:(NSString *)pageName;
- (void)synchronouslyGoBack;
- (void)synchronouslyGoForward;
Modified: trunk/Tools/TestWebKitAPI/cocoa/TestWKWebView.mm (290619 => 290620)
--- trunk/Tools/TestWebKitAPI/cocoa/TestWKWebView.mm 2022-02-28 23:48:00 UTC (rev 290619)
+++ trunk/Tools/TestWebKitAPI/cocoa/TestWKWebView.mm 2022-03-01 00:14:24 UTC (rev 290620)
@@ -183,6 +183,11 @@
return [self objectByEvaluatingJavaScript:@"Array.from(document.body.getElementsByTagName('*')).map(e => e.tagName)"];
}
+- (NSString *)selectedText
+{
+ return [self stringByEvaluatingJavaScript:@"getSelection().toString()"];
+}
+
- (void)expectElementTagsInOrder:(NSArray<NSString *> *)tagNames
{
auto remainingTags = adoptNS([tagNames mutableCopy]);
Modified: trunk/Tools/TestWebKitAPI/ios/TestUIMenuBuilder.h (290619 => 290620)
--- trunk/Tools/TestWebKitAPI/ios/TestUIMenuBuilder.h 2022-02-28 23:48:00 UTC (rev 290619)
+++ trunk/Tools/TestWebKitAPI/ios/TestUIMenuBuilder.h 2022-03-01 00:14:24 UTC (rev 290620)
@@ -31,7 +31,7 @@
@interface TestUIMenuBuilder : NSObject<UIMenuBuilder>
-- (BOOL)containsActionWithTitle:(NSString *)title;
+- (UIAction *)actionWithTitle:(NSString *)title;
- (void)reset;
@end
Modified: trunk/Tools/TestWebKitAPI/ios/TestUIMenuBuilder.mm (290619 => 290620)
--- trunk/Tools/TestWebKitAPI/ios/TestUIMenuBuilder.mm 2022-02-28 23:48:00 UTC (rev 290619)
+++ trunk/Tools/TestWebKitAPI/ios/TestUIMenuBuilder.mm 2022-03-01 00:14:24 UTC (rev 290620)
@@ -67,9 +67,9 @@
return nil;
}
-- (BOOL)containsActionWithTitle:(NSString *)title
+- (UIAction *)actionWithTitle:(NSString *)title
{
- return [self findMatching:^BOOL(UIMenuElement *element) {
+ return (UIAction *)[self findMatching:^BOOL(UIMenuElement *element) {
return [dynamic_objc_cast<UIAction>(element).title isEqual:title];
}];
}
Modified: trunk/Tools/TestWebKitAPI/ios/UIKitSPI.h (290619 => 290620)
--- trunk/Tools/TestWebKitAPI/ios/UIKitSPI.h 2022-02-28 23:48:00 UTC (rev 290619)
+++ trunk/Tools/TestWebKitAPI/ios/UIKitSPI.h 2022-03-01 00:14:24 UTC (rev 290620)
@@ -30,6 +30,7 @@
#if USE(APPLE_INTERNAL_SDK)
#import <UIKit/NSTextAlternatives.h>
+#import <UIKit/UIAction_Private.h>
#import <UIKit/UIApplication_Private.h>
#import <UIKit/UIBarButtonItemGroup_Private.h>
#import <UIKit/UICalloutBar.h>
@@ -201,6 +202,7 @@
@protocol UIWKInteractionViewProtocol
- (void)pasteWithCompletionHandler:(void (^)(void))completionHandler;
+- (void)requestRectsToEvadeForSelectionCommandsWithCompletionHandler:(void(^)(NSArray<NSValue *> *rects))completionHandler;
- (void)requestAutocorrectionRectsForString:(NSString *)input withCompletionHandler:(void (^)(UIWKAutocorrectionRects *rectsForInput))completionHandler;
- (void)requestAutocorrectionContextWithCompletionHandler:(void (^)(UIWKAutocorrectionContext *autocorrectionContext))completionHandler;
- (void)selectWordBackward;
@@ -269,6 +271,10 @@
- (void)lookup:(NSString *)textWithContext withRange:(NSRange)range fromRect:(CGRect)presentationRect;
@end
+@interface UIAction ()
+- (void)_performActionWithSender:(id)sender;
+@end
+
#endif // USE(APPLE_INTERNAL_SDK)
@interface UITextAutofillSuggestion ()
