Diff
Modified: trunk/LayoutTests/ChangeLog (279170 => 279171)
--- trunk/LayoutTests/ChangeLog 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/LayoutTests/ChangeLog 2021-06-23 16:59:34 UTC (rev 279171)
@@ -1,3 +1,16 @@
+2021-06-23 Andres Gonzalez <[email protected]>
+
+ Accessibility support for image text recognition.
+ https://bugs.webkit.org/show_bug.cgi?id=224280
+ rdar://76348740
+
+ Reviewed by Chris Fleizach.
+
+ * accessibility/image-link-expected.txt:
+ * accessibility/ios-simulator/image-overlay-elements-expected.txt: Added.
+ * accessibility/ios-simulator/image-overlay-elements.html: Added.
+ * accessibility/resources/green-400x400.png: Added.
+
2021-06-18 Mark Lam <[email protected]>
Add missing exception checks in ScriptModuleLoader and JSDOMPromiseDeferred.
Modified: trunk/LayoutTests/accessibility/image-link-expected.txt (279170 => 279171)
--- trunk/LayoutTests/accessibility/image-link-expected.txt 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/LayoutTests/accessibility/image-link-expected.txt 2021-06-23 16:59:34 UTC (rev 279171)
@@ -64,6 +64,7 @@
AXFocusableAncestor: <AXImage>
AXEditableAncestor: (null)
AXHighestEditableAncestor: (null)
+AXImageOverlayElements: (null)
AXEmbeddedImageDescription:
AXURL: LayoutTests/accessibility/resources/cake.png
AXElementBusy: 0
Added: trunk/LayoutTests/accessibility/ios-simulator/image-overlay-elements-expected.txt (0 => 279171)
--- trunk/LayoutTests/accessibility/ios-simulator/image-overlay-elements-expected.txt (rev 0)
+++ trunk/LayoutTests/accessibility/ios-simulator/image-overlay-elements-expected.txt 2021-06-23 16:59:34 UTC (rev 279171)
@@ -0,0 +1,7 @@
+PASS axImageOverlayElements.length is 2
+hello
+world
+PASS successfullyParsed is true
+
+TEST COMPLETE
+
Added: trunk/LayoutTests/accessibility/ios-simulator/image-overlay-elements.html (0 => 279171)
--- trunk/LayoutTests/accessibility/ios-simulator/image-overlay-elements.html (rev 0)
+++ trunk/LayoutTests/accessibility/ios-simulator/image-overlay-elements.html 2021-06-23 16:59:34 UTC (rev 279171)
@@ -0,0 +1,80 @@
+<!DOCTYPE html>
+<html>
+<head>
+<script src=""
+<script src=""
+<style>
+img {
+ position: absolute;
+ top: 0;
+ left: 0;
+}
+</style>
+</head>
+<body>
+
+<img id="image1" src=""
+
+<script>
+ window.jsTestIsAsync = true;
+
+ addEventListener("load", () => {
+ image = document.querySelector("img");
+ internals.installImageOverlay(image, [
+ {
+ topLeft : new DOMPointReadOnly(0, 0),
+ topRight : new DOMPointReadOnly(1, 0),
+ bottomRight : new DOMPointReadOnly(1, 0.5),
+ bottomLeft : new DOMPointReadOnly(0, 0.5),
+ children: [
+ {
+ text : "hello",
+ topLeft : new DOMPointReadOnly(0, 0),
+ topRight : new DOMPointReadOnly(1, 0),
+ bottomRight : new DOMPointReadOnly(1, 0.5),
+ bottomLeft : new DOMPointReadOnly(0, 0.5),
+ }
+ ],
+ },
+ {
+ topLeft : new DOMPointReadOnly(0, 0.5),
+ topRight : new DOMPointReadOnly(1, 0.5),
+ bottomRight : new DOMPointReadOnly(1, 1),
+ bottomLeft : new DOMPointReadOnly(0, 1),
+ children: [
+ {
+ text : "world",
+ topLeft : new DOMPointReadOnly(0, 0.5),
+ topRight : new DOMPointReadOnly(1, 0.5),
+ bottomRight : new DOMPointReadOnly(1, 1),
+ bottomLeft : new DOMPointReadOnly(0, 1),
+ }
+ ],
+ }
+ ]);
+
+ runTest();
+ });
+
+ function runTest() {
+ if (window.accessibilityController) {
+ setTimeout(async () => {
+ var axImage = accessibilityController.accessibleElementById("image1");
+ await waitFor(() => {
+ axImageOverlayElements = axImage.imageOverlayElements;
+ return axImageOverlayElements.length == 2;
+ });
+ shouldBe("axImageOverlayElements.length", "2");
+
+ for (let axElement of axImageOverlayElements) {
+ let textMarkerRange = axElement.textMarkerRangeForElement(axElement);
+ debug(axElement.stringForTextMarkerRange(textMarkerRange));
+ }
+
+ finishJSTest();
+ }, 0);
+ }
+ }
+</script>
+</body>
+</html>
Added: trunk/LayoutTests/accessibility/resources/green-400x400.png
(Binary files differ)
--- trunk/LayoutTests/accessibility/resources/green-400x400.png 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/LayoutTests/accessibility/resources/green-400x400.png 2021-06-23 16:59:34 UTC (rev 279171)
Property changes on: trunk/LayoutTests/accessibility/resources/green-400x400.png
___________________________________________________________________
Added: svn:mime-type
+image/png
\ No newline at end of property
Modified: trunk/LayoutTests/platform/ios/TestExpectations (279170 => 279171)
--- trunk/LayoutTests/platform/ios/TestExpectations 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/LayoutTests/platform/ios/TestExpectations 2021-06-23 16:59:34 UTC (rev 279171)
@@ -2120,6 +2120,9 @@
# Enable Text marker tests for iOS
webkit.org/b/153292 accessibility/text-marker [ Pass ]
+# Skip accessibility text recognition tests on iOS 14 and earlier.
+accessibility/ios-simulator/image-overlay-elements.html
+
# More flaky tests (Sept 18, 2015)
fast/forms/select-element-focus-ring.html [ Failure Pass ]
Modified: trunk/Source/WebCore/ChangeLog (279170 => 279171)
--- trunk/Source/WebCore/ChangeLog 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/ChangeLog 2021-06-23 16:59:34 UTC (rev 279171)
@@ -1,3 +1,59 @@
+2021-06-23 Andres Gonzalez <[email protected]>
+
+ Accessibility support for image text recognition.
+ https://bugs.webkit.org/show_bug.cgi?id=224280
+ rdar://76348740
+
+ Reviewed by Chris Fleizach.
+
+ Test: accessibility/ios-simulator/image-overlay-elements.html
+
+ Exposes to accessibility clients the elements created by
+ WebPage::requestTextRecognition for static images. This allows clients
+ to present the recognized text in images to assistive technology users.
+ - Added a new AXObject subclass, AXImage, to encapsulate this functionality.
+ This class can be expanded to offload some of the image specific code
+ contained in AccessibilityRenderObject, AccessibilityNodeObject,
+ AccessibilityObject and others.
+ - Since requestTextRecognition is an async call, added an AXNotification
+ to notify clients when the image overlay elements are available for
+ consumption.
+
+ * Sources.txt:
+ * WebCore.xcodeproj/project.pbxproj:
+ * accessibility/AXImage.cpp: Added.
+ (WebCore::AXImage::AXImage):
+ (WebCore::AXImage::create):
+ (WebCore::AXImage::roleValue const):
+ (WebCore::AXImage::imageOverlayElements):
+ * accessibility/AXImage.h: Added.
+ * accessibility/AXLogger.cpp:
+ (WebCore::operator<<):
+ * accessibility/AXObjectCache.cpp:
+ (WebCore::isSimpleImage): Determines whether a given element is a static
+ image.
+ (WebCore::createFromRenderer): Instantiate an AXImage object when
+ appropriate.
+ * accessibility/AXObjectCache.h:
+ * accessibility/AccessibilityNodeObject.cpp:
+ (WebCore::AccessibilityNodeObject::canHaveChildren const): Images can
+ have children.
+ * accessibility/AccessibilityObject.h:
+ * accessibility/AccessibilityObjectInterface.h:
+ * accessibility/ios/AXObjectCacheIOS.mm:
+ (WebCore::AXObjectCache::notificationPlatformName):
+ * accessibility/ios/WebAccessibilityObjectWrapperIOS.mm:
+ (-[WebAccessibilityObjectWrapper accessibilityElements]):
+ (-[WebAccessibilityObjectWrapper accessibilityImageOverlayElements]):
+ * accessibility/isolatedtree/AXIsolatedObject.cpp:
+ (WebCore::AXIsolatedObject::isAXImageInstance const):
+ * accessibility/isolatedtree/AXIsolatedObject.h:
+ * accessibility/mac/AXObjectCacheMac.mm:
+ (WebCore::AXObjectCache::postPlatformNotification):
+ * accessibility/mac/WebAccessibilityObjectWrapperMac.mm:
+ (-[WebAccessibilityObjectWrapper ALLOW_DEPRECATED_IMPLEMENTATIONS_END]):
+ (-[WebAccessibilityObjectWrapper accessibilityAttributeValue:]):
+
2021-06-18 Mark Lam <[email protected]>
Add missing exception checks in ScriptModuleLoader and JSDOMPromiseDeferred.
Modified: trunk/Source/WebCore/Sources.txt (279170 => 279171)
--- trunk/Source/WebCore/Sources.txt 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/Sources.txt 2021-06-23 16:59:34 UTC (rev 279171)
@@ -442,6 +442,7 @@
Modules/webxr/XRInputSourcesChangeEvent.cpp @no-unify
Modules/webxr/XRReferenceSpaceEvent.cpp @no-unify
Modules/webxr/XRSessionEvent.cpp @no-unify
+accessibility/AXImage.cpp
accessibility/AXLogger.cpp
accessibility/AXObjectCache.cpp
accessibility/AccessibilityARIAGrid.cpp
Modified: trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj (279170 => 279171)
--- trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj 2021-06-23 16:59:34 UTC (rev 279171)
@@ -14659,6 +14659,8 @@
BEA807C70F714A0300524199 /* SelectionGeometry.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SelectionGeometry.h; sourceTree = "<group>"; };
BEF29EE91715DD0900C4B4C9 /* AudioTrackPrivate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AudioTrackPrivate.h; sourceTree = "<group>"; };
BEF29EEA1715DD0900C4B4C9 /* VideoTrackPrivate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = VideoTrackPrivate.h; sourceTree = "<group>"; };
+ C0400C7C261CB3DB00AF44FA /* AXImage.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AXImage.h; sourceTree = "<group>"; };
+ C0400C7E261CB3DB00AF44FA /* AXImage.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = AXImage.cpp; sourceTree = "<group>"; };
C046E1AB1208A9FE00BA2CF7 /* LocalizedStrings.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = LocalizedStrings.cpp; sourceTree = "<group>"; };
C0493EB7244699E4009AAC80 /* AXLogger.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AXLogger.h; sourceTree = "<group>"; };
C078376023DF253E00C9C5C8 /* AXIsolatedObjectMac.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; name = AXIsolatedObjectMac.mm; path = mac/AXIsolatedObjectMac.mm; sourceTree = "<group>"; };
@@ -19456,6 +19458,8 @@
A91C9FC01B659A6700AFFD54 /* AccessibilityTreeItem.cpp */,
A91C9FC11B659A6700AFFD54 /* AccessibilityTreeItem.h */,
A9EE448C210FC9AD00B6EAE6 /* AriaAttributes.idl */,
+ C0400C7E261CB3DB00AF44FA /* AXImage.cpp */,
+ C0400C7C261CB3DB00AF44FA /* AXImage.h */,
C08F13A1244766DC00AB989F /* AXLogger.cpp */,
C0493EB7244699E4009AAC80 /* AXLogger.h */,
2981CAAF131822EC00D12F2A /* AXObjectCache.cpp */,
Added: trunk/Source/WebCore/accessibility/AXImage.cpp (0 => 279171)
--- trunk/Source/WebCore/accessibility/AXImage.cpp (rev 0)
+++ trunk/Source/WebCore/accessibility/AXImage.cpp 2021-06-23 16:59:34 UTC (rev 279171)
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of Apple Inc. ("Apple") nor the names of
+ * its contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "AXImage.h"
+
+#include "AXLogger.h"
+#include "Chrome.h"
+#include "ChromeClient.h"
+
+namespace WebCore {
+
+AXImage::AXImage(RenderImage* renderer)
+ : AccessibilityRenderObject(renderer)
+{
+}
+
+Ref<AXImage> AXImage::create(RenderImage* renderer)
+{
+ return adoptRef(*new AXImage(renderer));
+}
+
+AccessibilityRole AXImage::roleValue() const
+{
+ auto ariaRole = ariaRoleAttribute();
+ if (ariaRole != AccessibilityRole::Unknown)
+ return ariaRole;
+
+ return AccessibilityRole::Image;
+}
+
+std::optional<AXCoreObject::AccessibilityChildrenVector> AXImage::imageOverlayElements()
+{
+ AXTRACE("AXImage::imageOverlayElements");
+
+ auto& children = this->children();
+ if (children.size())
+ return children;
+
+#if ENABLE(IMAGE_ANALYSIS)
+ auto* page = this->page();
+ if (!page)
+ return std::nullopt;
+
+ page->chrome().client().requestTextRecognition(*element(), [] (RefPtr<Element>&& imageOverlayHost) {
+ if (!imageOverlayHost)
+ return;
+
+ if (auto* axObjectCache = imageOverlayHost->document().existingAXObjectCache())
+ axObjectCache->postNotification(imageOverlayHost.get(), AXObjectCache::AXImageOverlayChanged);
+ });
+#endif
+
+ return std::nullopt;
+}
+
+} // namespace WebCore
Added: trunk/Source/WebCore/accessibility/AXImage.h (0 => 279171)
--- trunk/Source/WebCore/accessibility/AXImage.h (rev 0)
+++ trunk/Source/WebCore/accessibility/AXImage.h 2021-06-23 16:59:34 UTC (rev 279171)
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of Apple Inc. ("Apple") nor the names of
+ * its contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include "AccessibilityRenderObject.h"
+#include "RenderImage.h"
+
+namespace WebCore {
+
+class AXImage : public AccessibilityRenderObject {
+public:
+ static Ref<AXImage> create(RenderImage*);
+ virtual ~AXImage() = default;
+
+private:
+ explicit AXImage(RenderImage*);
+ bool isAXImageInstance() const override { return true; }
+
+ AccessibilityRole roleValue() const override;
+ std::optional<AccessibilityChildrenVector> imageOverlayElements() override;
+};
+
+} // namespace WebCore
+
+SPECIALIZE_TYPE_TRAITS_ACCESSIBILITY(AXImage, isAXImageInstance())
Modified: trunk/Source/WebCore/accessibility/AXLogger.cpp (279170 => 279171)
--- trunk/Source/WebCore/accessibility/AXLogger.cpp 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/AXLogger.cpp 2021-06-23 16:59:34 UTC (rev 279171)
@@ -372,6 +372,9 @@
case AXObjectCache::AXNotification::AXIdAttributeChanged:
stream << "AXIdAttributeChanged";
break;
+ case AXObjectCache::AXNotification::AXImageOverlayChanged:
+ stream << "AXImageOverlayChanged";
+ break;
case AXObjectCache::AXNotification::AXLanguageChanged:
stream << "AXLanguageChanged";
break;
Modified: trunk/Source/WebCore/accessibility/AXObjectCache.cpp (279170 => 279171)
--- trunk/Source/WebCore/accessibility/AXObjectCache.cpp 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/AXObjectCache.cpp 2021-06-23 16:59:34 UTC (rev 279171)
@@ -511,6 +511,28 @@
return SpaceSplitString(roleValue, true).contains(role);
}
+static bool isSimpleImage(const RenderObject& renderer)
+{
+ if (!is<RenderImage>(renderer))
+ return false;
+
+ // Exclude ImageButtons because they are treated as buttons, not as images.
+ auto* node = renderer.node();
+ if (is<HTMLInputElement>(node))
+ return false;
+
+ // ImageMaps are not simple images.
+ if (downcast<RenderImage>(renderer).imageMap()
+ || (is<HTMLImageElement>(node) && downcast<HTMLImageElement>(node)->hasAttributeWithoutSynchronization(usemapAttr)))
+ return false;
+
+ // Exclude video and audio elements.
+ if (is<HTMLMediaElement>(node))
+ return false;
+
+ return true;
+}
+
static Ref<AccessibilityObject> createFromRenderer(RenderObject* renderer)
{
// FIXME: How could renderer->node() ever not be an Element?
@@ -550,6 +572,9 @@
if (is<SVGElement>(node))
return AccessibilitySVGElement::create(renderer);
+ if (isSimpleImage(*renderer))
+ return AXImage::create(downcast<RenderImage>(renderer));
+
#if ENABLE(MATHML)
// The mfenced element creates anonymous RenderMathMLOperators which should be treated
// as MathML elements and assigned the MathElementRole so that platform logic regarding
Modified: trunk/Source/WebCore/accessibility/AXObjectCache.h (279170 => 279171)
--- trunk/Source/WebCore/accessibility/AXObjectCache.h 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/AXObjectCache.h 2021-06-23 16:59:34 UTC (rev 279171)
@@ -284,6 +284,7 @@
AXFocusedUIElementChanged,
AXFrameLoadComplete,
AXIdAttributeChanged,
+ AXImageOverlayChanged,
AXLanguageChanged,
AXLayoutComplete,
AXLoadComplete,
Modified: trunk/Source/WebCore/accessibility/AccessibilityNodeObject.cpp (279170 => 279171)
--- trunk/Source/WebCore/accessibility/AccessibilityNodeObject.cpp 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/AccessibilityNodeObject.cpp 2021-06-23 16:59:34 UTC (rev 279171)
@@ -378,7 +378,6 @@
// Elements that should not have children
switch (roleValue()) {
- case AccessibilityRole::Image:
case AccessibilityRole::Button:
case AccessibilityRole::PopUpButton:
case AccessibilityRole::CheckBox:
Modified: trunk/Source/WebCore/accessibility/AccessibilityObject.h (279170 => 279171)
--- trunk/Source/WebCore/accessibility/AccessibilityObject.h 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/AccessibilityObject.h 2021-06-23 16:59:34 UTC (rev 279171)
@@ -93,6 +93,7 @@
bool isAccessibilityRenderObject() const override { return false; }
bool isAccessibilityScrollbar() const override { return false; }
bool isAccessibilityScrollViewInstance() const override { return false; }
+ bool isAXImageInstance() const override { return false; }
bool isAccessibilitySVGRoot() const override { return false; }
bool isAccessibilitySVGElement() const override { return false; }
bool isAccessibilityTableInstance() const override { return false; }
@@ -399,6 +400,7 @@
String brailleLabel() const override { return getAttribute(HTMLNames::aria_braillelabelAttr); }
String brailleRoleDescription() const override { return getAttribute(HTMLNames::aria_brailleroledescriptionAttr); }
String embeddedImageDescription() const override;
+ std::optional<AccessibilityChildrenVector> imageOverlayElements() override { return std::nullopt; }
// Abbreviations
String expandedTextValue() const override { return String(); }
Modified: trunk/Source/WebCore/accessibility/AccessibilityObjectInterface.h (279170 => 279171)
--- trunk/Source/WebCore/accessibility/AccessibilityObjectInterface.h 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/AccessibilityObjectInterface.h 2021-06-23 16:59:34 UTC (rev 279171)
@@ -791,6 +791,7 @@
virtual bool isAccessibilityRenderObject() const = 0;
virtual bool isAccessibilityScrollbar() const = 0;
virtual bool isAccessibilityScrollViewInstance() const = 0;
+ virtual bool isAXImageInstance() const = 0;
virtual bool isAccessibilitySVGRoot() const = 0;
virtual bool isAccessibilitySVGElement() const = 0;
virtual bool isAccessibilityTableInstance() const = 0;
@@ -1014,6 +1015,7 @@
virtual String brailleLabel() const = 0;
virtual String brailleRoleDescription() const = 0;
virtual String embeddedImageDescription() const = 0;
+ virtual std::optional<AccessibilityChildrenVector> imageOverlayElements() = 0;
virtual bool supportsARIAOwns() const = 0;
virtual bool isActiveDescendantOfFocusedContainer() const = 0;
Modified: trunk/Source/WebCore/accessibility/ios/AXObjectCacheIOS.mm (279170 => 279171)
--- trunk/Source/WebCore/accessibility/ios/AXObjectCacheIOS.mm 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/ios/AXObjectCacheIOS.mm 2021-06-23 16:59:34 UTC (rev 279171)
@@ -51,6 +51,9 @@
case AXFocusedUIElementChanged:
name = "AXFocusChanged";
break;
+ case AXImageOverlayChanged:
+ name = "AXImageOverlayChanged";
+ break;
case AXPageScrolled:
name = "AXPageScrolled";
break;
Modified: trunk/Source/WebCore/accessibility/ios/WebAccessibilityObjectWrapperIOS.mm (279170 => 279171)
--- trunk/Source/WebCore/accessibility/ios/WebAccessibilityObjectWrapperIOS.mm 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/ios/WebAccessibilityObjectWrapperIOS.mm 2021-06-23 16:59:34 UTC (rev 279171)
@@ -72,6 +72,7 @@
- (NSInteger)accessibilityElementCount;
- (id)accessibilityElementAtIndex:(NSInteger)index;
- (NSInteger)indexOfAccessibilityElement:(id)element;
+- (NSArray *)accessibilityElements;
@end
@interface WebAccessibilityObjectWrapper (AccessibilityPrivate)
@@ -429,6 +430,29 @@
cache->stopCachingComputedObjectAttributes();
}
+- (NSArray *)accessibilityElements
+{
+ if (![self _prepareAccessibilityCall])
+ return nil;
+
+ if ([self isAttachment]) {
+ if (id attachmentView = [self attachmentView])
+ return [attachmentView accessibilityElements];
+ }
+
+ auto array = adoptNS([[NSMutableArray alloc] init]);
+ for (const auto& child : self.axBackingObject->children()) {
+ auto* wrapper = child->wrapper();
+ if (child->isAttachment()) {
+ if (id attachmentView = [wrapper attachmentView])
+ [array addObject:attachmentView];
+ } else
+ [array addObject:wrapper];
+ }
+
+ return array.autorelease();
+}
+
- (NSInteger)accessibilityElementCount
{
if (![self _prepareAccessibilityCall])
@@ -1854,6 +1878,15 @@
return self.axBackingObject->embeddedImageDescription();
}
+- (NSArray *)accessibilityImageOverlayElements
+{
+ if (![self _prepareAccessibilityCall])
+ return nil;
+
+ auto imageOverlayElements = self.axBackingObject->imageOverlayElements();
+ return imageOverlayElements ? accessibleElementsForObjects(*imageOverlayElements) : nil;
+}
+
- (NSString *)accessibilityLinkRelationshipType
{
if (![self _prepareAccessibilityCall])
Modified: trunk/Source/WebCore/accessibility/isolatedtree/AXIsolatedObject.cpp (279170 => 279171)
--- trunk/Source/WebCore/accessibility/isolatedtree/AXIsolatedObject.cpp 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/isolatedtree/AXIsolatedObject.cpp 2021-06-23 16:59:34 UTC (rev 279171)
@@ -1258,6 +1258,12 @@
return false;
}
+bool AXIsolatedObject::isAXImageInstance() const
+{
+ ASSERT_NOT_REACHED();
+ return false;
+}
+
bool AXIsolatedObject::isAccessibilitySVGRoot() const
{
ASSERT_NOT_REACHED();
Modified: trunk/Source/WebCore/accessibility/isolatedtree/AXIsolatedObject.h (279170 => 279171)
--- trunk/Source/WebCore/accessibility/isolatedtree/AXIsolatedObject.h 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/isolatedtree/AXIsolatedObject.h 2021-06-23 16:59:34 UTC (rev 279171)
@@ -272,6 +272,7 @@
String brailleLabel() const override { return stringAttributeValue(AXPropertyName::BrailleLabel); }
String brailleRoleDescription() const override { return stringAttributeValue(AXPropertyName::BrailleRoleDescription); }
String embeddedImageDescription() const override { return stringAttributeValue(AXPropertyName::EmbeddedImageDescription); }
+ std::optional<AccessibilityChildrenVector> imageOverlayElements() override { return std::nullopt; }
String computedRoleString() const override { return stringAttributeValue(AXPropertyName::ComputedRoleString); }
bool isValueAutofilled() const override { return boolAttributeValue(AXPropertyName::IsValueAutofilled); }
@@ -465,6 +466,7 @@
bool isAccessibilityRenderObject() const override;
bool isAccessibilityScrollbar() const override;
bool isAccessibilityScrollViewInstance() const override;
+ bool isAXImageInstance() const override;
bool isAccessibilitySVGRoot() const override;
bool isAccessibilitySVGElement() const override;
bool isAccessibilityTableInstance() const override;
Modified: trunk/Source/WebCore/accessibility/mac/AXObjectCacheMac.mm (279170 => 279171)
--- trunk/Source/WebCore/accessibility/mac/AXObjectCacheMac.mm 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/mac/AXObjectCacheMac.mm 2021-06-23 16:59:34 UTC (rev 279171)
@@ -319,6 +319,9 @@
case AXFocusedUIElementChanged:
macNotification = NSAccessibilityFocusedUIElementChangedNotification;
break;
+ case AXImageOverlayChanged:
+ macNotification = @"AXImageOverlayChanged";
+ break;
case AXLayoutComplete:
macNotification = @"AXLayoutComplete";
break;
Modified: trunk/Source/WebCore/accessibility/mac/WebAccessibilityObjectWrapperMac.mm (279170 => 279171)
--- trunk/Source/WebCore/accessibility/mac/WebAccessibilityObjectWrapperMac.mm 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Source/WebCore/accessibility/mac/WebAccessibilityObjectWrapperMac.mm 2021-06-23 16:59:34 UTC (rev 279171)
@@ -543,6 +543,10 @@
#define NSAccessibilityEmbeddedImageDescriptionAttribute @"AXEmbeddedImageDescription"
#endif
+#ifndef NSAccessibilityImageOverlayElementsAttribute
+#define NSAccessibilityImageOverlayElementsAttribute @"AXImageOverlayElements"
+#endif
+
extern "C" AXUIElementRef NSAccessibilityCreateAXUIElementRef(id element);
@implementation WebAccessibilityObjectWrapper
@@ -1560,8 +1564,9 @@
[tempArray addObject:NSAccessibilityVerticalScrollBarAttribute];
return tempArray;
}());
- static auto imageAttrs = makeNeverDestroyed([] {
+ static auto imageAttrs = makeNeverDestroyed([] {
auto tempArray = adoptNS([[NSMutableArray alloc] initWithArray:attributes.get().get()]);
+ [tempArray addObject:NSAccessibilityImageOverlayElementsAttribute];
[tempArray addObject:NSAccessibilityEmbeddedImageDescriptionAttribute];
[tempArray addObject:NSAccessibilityURLAttribute];
return tempArray;
@@ -2366,6 +2371,12 @@
return [self position];
if ([attributeName isEqualToString:NSAccessibilityPathAttribute])
return [self path];
+
+ if ([attributeName isEqualToString:NSAccessibilityImageOverlayElementsAttribute]) {
+ auto imageOverlayElements = backingObject->imageOverlayElements();
+ return imageOverlayElements ? convertToNSArray(*imageOverlayElements) : nil;
+ }
+
if ([attributeName isEqualToString:NSAccessibilityEmbeddedImageDescriptionAttribute])
return backingObject->embeddedImageDescription();
Modified: trunk/Tools/ChangeLog (279170 => 279171)
--- trunk/Tools/ChangeLog 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Tools/ChangeLog 2021-06-23 16:59:34 UTC (rev 279171)
@@ -1,3 +1,22 @@
+2021-06-23 Andres Gonzalez <[email protected]>
+
+ Accessibility support for image text recognition.
+ https://bugs.webkit.org/show_bug.cgi?id=224280
+ rdar://76348740
+
+ Reviewed by Chris Fleizach.
+
+ * WebKitTestRunner/InjectedBundle/AccessibilityUIElement.cpp:
+ (WTR::AccessibilityUIElement::children const):
+ * WebKitTestRunner/InjectedBundle/AccessibilityUIElement.h:
+ * WebKitTestRunner/InjectedBundle/Bindings/AccessibilityUIElement.idl:
+ * WebKitTestRunner/InjectedBundle/ios/AccessibilityUIElementIOS.mm:
+ (WTR::AccessibilityUIElement::children const):
+ (WTR::AccessibilityUIElement::imageOverlayElements const):
+ * WebKitTestRunner/InjectedBundle/mac/AccessibilityUIElementMac.mm:
+ (WTR::AccessibilityUIElement::children const):
+ (WTR::AccessibilityUIElement::imageOverlayElements const):
+
2021-06-23 Chris Dumez <[email protected]>
Update WPT tools from upstream
Modified: trunk/Tools/WebKitTestRunner/InjectedBundle/AccessibilityUIElement.cpp (279170 => 279171)
--- trunk/Tools/WebKitTestRunner/InjectedBundle/AccessibilityUIElement.cpp 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Tools/WebKitTestRunner/InjectedBundle/AccessibilityUIElement.cpp 2021-06-23 16:59:34 UTC (rev 279171)
@@ -127,8 +127,10 @@
RefPtr<AccessibilityTextMarker> AccessibilityUIElement::previousSentenceStartTextMarkerForTextMarker(AccessibilityTextMarker*) { return nullptr; }
RefPtr<AccessibilityTextMarkerRange> AccessibilityUIElement::misspellingTextMarkerRange(AccessibilityTextMarkerRange*, bool) { return nullptr; }
void AccessibilityUIElement::dismiss() { }
+JSValueRef AccessibilityUIElement::children() const { return { }; }
JSValueRef AccessibilityUIElement::detailsElements() const { return { }; }
JSValueRef AccessibilityUIElement::errorMessageElements() const { return { }; }
+JSValueRef AccessibilityUIElement::imageOverlayElements() const { return { }; }
JSRetainPtr<JSStringRef> AccessibilityUIElement::embeddedImageDescription() const { return nullptr; }
#endif
Modified: trunk/Tools/WebKitTestRunner/InjectedBundle/AccessibilityUIElement.h (279170 => 279171)
--- trunk/Tools/WebKitTestRunner/InjectedBundle/AccessibilityUIElement.h 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Tools/WebKitTestRunner/InjectedBundle/AccessibilityUIElement.h 2021-06-23 16:59:34 UTC (rev 279171)
@@ -83,6 +83,7 @@
JSRetainPtr<JSStringRef> domIdentifier() const;
RefPtr<AccessibilityUIElement> elementAtPoint(int x, int y);
+ JSValueRef children() const;
RefPtr<AccessibilityUIElement> childAtIndex(unsigned);
unsigned indexOfChild(AccessibilityUIElement*);
int childrenCount();
@@ -196,7 +197,8 @@
JSRetainPtr<JSStringRef> url();
JSRetainPtr<JSStringRef> classList() const;
JSRetainPtr<JSStringRef> embeddedImageDescription() const;
-
+ JSValueRef imageOverlayElements() const;
+
// CSS3-speech properties.
JSRetainPtr<JSStringRef> speakAs();
Modified: trunk/Tools/WebKitTestRunner/InjectedBundle/Bindings/AccessibilityUIElement.idl (279170 => 279171)
--- trunk/Tools/WebKitTestRunner/InjectedBundle/Bindings/AccessibilityUIElement.idl 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Tools/WebKitTestRunner/InjectedBundle/Bindings/AccessibilityUIElement.idl 2021-06-23 16:59:34 UTC (rev 279171)
@@ -110,6 +110,7 @@
readonly attribute DOMString ariaDropEffects;
readonly attribute DOMString classList;
readonly attribute DOMString embeddedImageDescription;
+ readonly attribute object imageOverlayElements;
readonly attribute long x;
readonly attribute long y;
@@ -118,6 +119,7 @@
readonly attribute long clickPointX;
readonly attribute long clickPointY;
+ readonly attribute object children;
readonly attribute long childrenCount;
readonly attribute long selectedChildrenCount;
readonly attribute long rowCount;
Modified: trunk/Tools/WebKitTestRunner/InjectedBundle/ios/AccessibilityUIElementIOS.mm (279170 => 279171)
--- trunk/Tools/WebKitTestRunner/InjectedBundle/ios/AccessibilityUIElementIOS.mm 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Tools/WebKitTestRunner/InjectedBundle/ios/AccessibilityUIElementIOS.mm 2021-06-23 16:59:34 UTC (rev 279171)
@@ -102,6 +102,7 @@
- (id)_accessibilityLandmarkAncestor;
- (id)_accessibilityListAncestor;
- (id)_accessibilityPhotoDescription;
+- (NSArray *)accessibilityImageOverlayElements;
// TextMarker related
- (NSArray *)textMarkerRange;
@@ -215,6 +216,11 @@
{
}
+JSValueRef AccessibilityUIElement::children() const
+{
+ return makeJSArray(makeVector<RefPtr<AccessibilityUIElement>>([m_element accessibilityElements]));
+}
+
void AccessibilityUIElement::getChildren(Vector<RefPtr<AccessibilityUIElement> >& elementVector)
{
NSInteger childCount = [m_element accessibilityElementCount];
@@ -1101,6 +1107,11 @@
return concatenateAttributeAndValue(@"AXEmbeddedImageDescription", [m_element _accessibilityPhotoDescription]);
}
+JSValueRef AccessibilityUIElement::imageOverlayElements() const
+{
+ return makeJSArray(makeVector<RefPtr<AccessibilityUIElement>>([m_element accessibilityImageOverlayElements]));
+}
+
bool AccessibilityUIElement::isCollapsed() const
{
return false;
Modified: trunk/Tools/WebKitTestRunner/InjectedBundle/mac/AccessibilityUIElementMac.mm (279170 => 279171)
--- trunk/Tools/WebKitTestRunner/InjectedBundle/mac/AccessibilityUIElementMac.mm 2021-06-23 16:25:52 UTC (rev 279170)
+++ trunk/Tools/WebKitTestRunner/InjectedBundle/mac/AccessibilityUIElementMac.mm 2021-06-23 16:59:34 UTC (rev 279171)
@@ -349,6 +349,15 @@
elements = makeVector<RefPtr<AccessibilityUIElement>>(value);
}
+JSValueRef AccessibilityUIElement::children() const
+{
+ BEGIN_AX_OBJC_EXCEPTIONS
+ return makeJSArray(makeVector<RefPtr<AccessibilityUIElement>>(attributeValue(m_element.get(), NSAccessibilityChildrenAttribute)));
+ END_AX_OBJC_EXCEPTIONS
+
+ return nullptr;
+}
+
void AccessibilityUIElement::getChildren(Vector<RefPtr<AccessibilityUIElement> >& elementVector)
{
elementVector = makeVector<RefPtr<AccessibilityUIElement>>(attributeValue(m_element.get(), NSAccessibilityChildrenAttribute));
@@ -1674,6 +1683,15 @@
return nullptr;
}
+JSValueRef AccessibilityUIElement::imageOverlayElements() const
+{
+ BEGIN_AX_OBJC_EXCEPTIONS
+ return makeJSArray(makeVector<RefPtr<AccessibilityUIElement>>(attributeValue(m_element.get(), @"AXImageOverlayElements")));
+ END_AX_OBJC_EXCEPTIONS
+
+ return nullptr;
+}
+
bool AccessibilityUIElement::isIgnored() const
{
BOOL result = NO;
