From 19be572305b883c7a47b9a270271c97f3d2c93db Mon Sep 17 00:00:00 2001 From: Chris Yang Date: Tue, 26 Mar 2019 16:04:42 -0700 Subject: [PATCH] Add a11y support for embedded iOS platform view (flutter/engine#8156) Follow up the framework change in flutter/flutter#29304. Inject the accessibility element tree in the semantic node if the node is for platform views. flutter/flutter#29302 --- .../lib/ui/semantics/semantics_node.cc | 6 ++ .../flutter/lib/ui/semantics/semantics_node.h | 3 + .../framework/Source/FlutterPlatformViews.mm | 7 ++ .../Source/FlutterPlatformViews_Internal.h | 7 ++ .../framework/Source/accessibility_bridge.h | 33 ++++++++- .../framework/Source/accessibility_bridge.mm | 68 ++++++++++++++++++- .../platform/darwin/ios/platform_view_ios.mm | 6 +- 7 files changed, 124 insertions(+), 6 deletions(-) diff --git a/engine/src/flutter/lib/ui/semantics/semantics_node.cc b/engine/src/flutter/lib/ui/semantics/semantics_node.cc index cb655bdde8..fd44758aa3 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_node.cc +++ b/engine/src/flutter/lib/ui/semantics/semantics_node.cc @@ -8,6 +8,8 @@ namespace blink { +constexpr int32_t kMinPlatfromViewId = -1; + SemanticsNode::SemanticsNode() = default; SemanticsNode::SemanticsNode(const SemanticsNode& other) = default; @@ -22,4 +24,8 @@ bool SemanticsNode::HasFlag(SemanticsFlags flag) { return (flags & static_cast(flag)) != 0; } +bool SemanticsNode::IsPlatformViewNode() const { + return platformViewId > kMinPlatfromViewId; +} + } // namespace blink diff --git a/engine/src/flutter/lib/ui/semantics/semantics_node.h b/engine/src/flutter/lib/ui/semantics/semantics_node.h index 16a87a32d9..6d570b42c7 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_node.h +++ b/engine/src/flutter/lib/ui/semantics/semantics_node.h @@ -81,6 +81,9 @@ struct SemanticsNode { bool HasAction(SemanticsAction action); bool HasFlag(SemanticsFlags flag); + // Whether this node is for embeded platform views. + bool IsPlatformViewNode() const; + int32_t id = 0; int32_t flags = 0; int32_t actions = 0; diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm index 4eb773532a..d3c42ed446 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm @@ -165,6 +165,13 @@ void FlutterPlatformViewsController::PrerollCompositeEmbeddedView(int view_id) { composition_order_.push_back(view_id); } +NSObject* FlutterPlatformViewsController::GetPlatformViewByID(int view_id) { + if (views_.empty()) { + return nil; + } + return views_[view_id].get(); +} + std::vector FlutterPlatformViewsController::GetCurrentCanvases() { std::vector canvases; for (size_t i = 0; i < composition_order_.size(); i++) { diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews_Internal.h b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews_Internal.h index 118822c3b7..a0ebbac98b 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews_Internal.h +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews_Internal.h @@ -58,6 +58,13 @@ class FlutterPlatformViewsController { void PrerollCompositeEmbeddedView(int view_id); + // Returns the `FlutterPlatformView` object associated with the view_id. + // + // If the `FlutterPlatformViewsController` does not contain any `FlutterPlatformView` object or + // a `FlutterPlatformView` object asscociated with the view_id cannot be found, the method returns + // nil. + NSObject* GetPlatformViewByID(int view_id); + std::vector GetCurrentCanvases(); SkCanvas* CompositeEmbeddedView(int view_id, const flow::EmbeddedViewParams& params); diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h index 465abc5035..f48064a258 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h @@ -27,6 +27,8 @@ namespace shell { class AccessibilityBridge; } // namespace shell +@class FlutterPlatformViewSemanticsContainer; + /** * A node in the iOS semantics tree. */ @@ -71,6 +73,11 @@ class AccessibilityBridge; */ @property(nonatomic, strong) NSMutableArray* children; +/** + * Used if this SemanticsObject is for a platform view. + */ +@property(strong, nonatomic) FlutterPlatformViewSemanticsContainer* platformViewSemanticsContainer; + - (BOOL)nodeWillCauseLayoutChange:(const blink::SemanticsNode*)node; #pragma mark - Designated initializers @@ -108,12 +115,31 @@ class AccessibilityBridge; @interface FlutterSemanticsObject : SemanticsObject @end +/** + * Designated to act as an accessibility container of a platform view. + * + * This object does not take any accessibility actions on its own, nor has any accessibility + * label/value/trait/hint... on its own. The accessibility data will be handled by the platform + * view. + * + * See also: + * * `SemanticsObject` for the other type of semantics objects. + * * `FlutterSemanticsObject` for default implementation of `SemanticsObject`. + */ +@interface FlutterPlatformViewSemanticsContainer : UIAccessibilityElement + +- (instancetype)init __attribute__((unavailable("Use initWithAccessibilityContainer: instead"))); + +@end + namespace shell { class PlatformViewIOS; class AccessibilityBridge final { public: - AccessibilityBridge(UIView* view, PlatformViewIOS* platform_view); + AccessibilityBridge(UIView* view, + PlatformViewIOS* platform_view, + FlutterPlatformViewsController* platform_views_controller); ~AccessibilityBridge(); void UpdateSemantics(blink::SemanticsNodeUpdates nodes, @@ -129,6 +155,10 @@ class AccessibilityBridge final { fml::WeakPtr GetWeakPtr(); + FlutterPlatformViewsController* GetPlatformViewsController() const { + return platform_views_controller_; + }; + void clearState(); private: @@ -139,6 +169,7 @@ class AccessibilityBridge final { UIView* view_; PlatformViewIOS* platform_view_; + FlutterPlatformViewsController* platform_views_controller_; fml::scoped_nsobject> objects_; fml::scoped_nsprotocol accessibility_channel_; fml::WeakPtrFactory weak_factory_; diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm index 7fe516ac80..39bfce29fb 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm @@ -11,6 +11,7 @@ #import #include "flutter/fml/logging.h" +#include "flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews_Internal.h" #include "flutter/shell/platform/darwin/ios/platform_view_ios.h" namespace { @@ -127,6 +128,7 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection( [_children release]; _parent = nil; _container.get().semanticsObject = nil; + [_platformViewSemanticsContainer release]; [super dealloc]; } @@ -152,6 +154,9 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection( } - (BOOL)hasChildren { + if (_node.IsPlatformViewNode()) { + return YES; + } return [self.children count] != 0; } @@ -165,6 +170,7 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection( // We enforce in the framework that no other useful semantics are merged with these nodes. if ([self node].HasFlag(blink::SemanticsFlags::kScopesRoute)) return false; + return ([self node].flags != 0 && [self node].flags != static_cast(blink::SemanticsFlags::kIsHidden)) || ![self node].label.empty() || ![self node].value.empty() || ![self node].hint.empty() || @@ -396,6 +402,25 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection( @end +@implementation FlutterPlatformViewSemanticsContainer + +// Method declared as unavailable in the interface +- (instancetype)init { + [self release]; + [super doesNotRecognizeSelector:_cmd]; + return nil; +} + +- (instancetype)initWithAccessibilityContainer:(id)container { + FML_CHECK(container); + if (self = [super initWithAccessibilityContainer:container]) { + self.isAccessibilityElement = NO; + } + return self; +} + +@end + @implementation SemanticsObjectContainer { SemanticsObject* _semanticsObject; fml::WeakPtr _bridge; @@ -426,7 +451,12 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection( #pragma mark - UIAccessibilityContainer overrides - (NSInteger)accessibilityElementCount { - return [[_semanticsObject children] count] + 1; + NSInteger count = [[_semanticsObject children] count] + 1; + // Need to create an additional child that acts as accessibility container for the platform view. + if (_semanticsObject.node.IsPlatformViewNode()) { + count++; + } + return count; } - (nullable id)accessibilityElementAtIndex:(NSInteger)index { @@ -435,7 +465,16 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection( if (index == 0) { return _semanticsObject; } + + // Return the additional child acts as a container of platform view. The + // platformViewSemanticsContainer was created and cached in the updateSemantics path. + if (_semanticsObject.node.IsPlatformViewNode() && index == [self accessibilityElementCount] - 1) { + FML_CHECK(_semanticsObject.platformViewSemanticsContainer != nil); + return _semanticsObject.platformViewSemanticsContainer; + } + SemanticsObject* child = [_semanticsObject children][index - 1]; + if ([child hasChildren]) return [child accessibilityContainer]; return child; @@ -444,6 +483,12 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection( - (NSInteger)indexOfAccessibilityElement:(id)element { if (element == _semanticsObject) return 0; + + // FlutterPlatformViewSemanticsContainer is always the last element of its parent. + if ([element isKindOfClass:[FlutterPlatformViewSemanticsContainer class]]) { + return [self accessibilityElementCount] - 1; + } + NSMutableArray* children = [_semanticsObject children]; for (size_t i = 0; i < [children count]; i++) { SemanticsObject* child = children[i]; @@ -485,9 +530,12 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection( namespace shell { -AccessibilityBridge::AccessibilityBridge(UIView* view, PlatformViewIOS* platform_view) +AccessibilityBridge::AccessibilityBridge(UIView* view, + PlatformViewIOS* platform_view, + FlutterPlatformViewsController* platform_views_controller) : view_(view), platform_view_(platform_view), + platform_views_controller_(platform_views_controller), objects_([[NSMutableDictionary alloc] init]), weak_factory_(this), previous_route_id_(0), @@ -525,7 +573,7 @@ void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes, layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node]; scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node]; [object setSemanticsNode:&node]; - const NSUInteger newChildCount = node.childrenInTraversalOrder.size(); + NSUInteger newChildCount = node.childrenInTraversalOrder.size(); NSMutableArray* newChildren = [[[NSMutableArray alloc] initWithCapacity:newChildCount] autorelease]; for (NSUInteger i = 0; i < newChildCount; ++i) { @@ -555,6 +603,20 @@ void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes, } object.accessibilityCustomActions = accessibilityCustomActions; } + + if (object.node.IsPlatformViewNode()) { + shell::FlutterPlatformViewsController* controller = GetPlatformViewsController(); + if (controller) { + object.platformViewSemanticsContainer = [[FlutterPlatformViewSemanticsContainer alloc] + initWithAccessibilityContainer:[object accessibilityContainer]]; + UIView* platformView = [controller->GetPlatformViewByID(object.node.platformViewId) view]; + if (platformView) { + object.platformViewSemanticsContainer.accessibilityElements = @[ platformView ]; + } + } + } else if (object.platformViewSemanticsContainer) { + [object.platformViewSemanticsContainer release]; + } } SemanticsObject* root = objects_.get()[@(kRootNodeId)]; diff --git a/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm b/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm index 6f138edcfd..c0b988be88 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm @@ -50,7 +50,8 @@ void PlatformViewIOS::SetOwnerViewController(fml::WeakPtr if (accessibility_bridge_) { accessibility_bridge_.reset( - new AccessibilityBridge(static_cast(owner_controller_.get().view), this)); + new AccessibilityBridge(static_cast(owner_controller_.get().view), this, + [owner_controller.get() platformViewsController])); } // Do not call `NotifyCreated()` here - let FlutterViewController take care // of that when its Viewport is sized. If `NotifyCreated()` is called here, @@ -96,7 +97,8 @@ void PlatformViewIOS::SetSemanticsEnabled(bool enabled) { } if (enabled && !accessibility_bridge_) { accessibility_bridge_ = std::make_unique( - static_cast(owner_controller_.get().view), this); + static_cast(owner_controller_.get().view), this, + [owner_controller_.get() platformViewsController]); } else if (!enabled && accessibility_bridge_) { accessibility_bridge_.reset(); }