Add a11y support for embedded iOS platform view (flutter/engine#8156)

Follow up the framework change in flutter/flutter#29304.
Inject the accessibility element tree in the semantic node if the node is for platform views.

flutter/flutter#29302

engine/src/flutter/lib/ui/semantics/semantics_node.cc

@@ -8,6 +8,8 @@
 
 namespace blink {
 
+constexpr int32_t kMinPlatfromViewId = -1;
+
 SemanticsNode::SemanticsNode() = default;
 
 SemanticsNode::SemanticsNode(const SemanticsNode& other) = default;
@@ -22,4 +24,8 @@ bool SemanticsNode::HasFlag(SemanticsFlags flag) {
   return (flags & static_cast<int32_t>(flag)) != 0;
 }
 
+bool SemanticsNode::IsPlatformViewNode() const {
+  return platformViewId > kMinPlatfromViewId;
+}
+
 }  // namespace blink

engine/src/flutter/lib/ui/semantics/semantics_node.h

@@ -81,6 +81,9 @@ struct SemanticsNode {
   bool HasAction(SemanticsAction action);
   bool HasFlag(SemanticsFlags flag);
 
+  // Whether this node is for embeded platform views.
+  bool IsPlatformViewNode() const;
+
   int32_t id = 0;
   int32_t flags = 0;
   int32_t actions = 0;

engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm

@@ -165,6 +165,13 @@ void FlutterPlatformViewsController::PrerollCompositeEmbeddedView(int view_id) {
   composition_order_.push_back(view_id);
 }
 
+NSObject<FlutterPlatformView>* FlutterPlatformViewsController::GetPlatformViewByID(int view_id) {
+  if (views_.empty()) {
+    return nil;
+  }
+  return views_[view_id].get();
+}
+
 std::vector<SkCanvas*> FlutterPlatformViewsController::GetCurrentCanvases() {
   std::vector<SkCanvas*> canvases;
   for (size_t i = 0; i < composition_order_.size(); i++) {

engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews_Internal.h

@@ -58,6 +58,13 @@ class FlutterPlatformViewsController {
 
   void PrerollCompositeEmbeddedView(int view_id);
 
+  // Returns the `FlutterPlatformView` object associated with the view_id.
+  //
+  // If the `FlutterPlatformViewsController` does not contain any `FlutterPlatformView` object or
+  // a `FlutterPlatformView` object asscociated with the view_id cannot be found, the method returns
+  // nil.
+  NSObject<FlutterPlatformView>* GetPlatformViewByID(int view_id);
+
   std::vector<SkCanvas*> GetCurrentCanvases();
 
   SkCanvas* CompositeEmbeddedView(int view_id, const flow::EmbeddedViewParams& params);

engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h

@@ -27,6 +27,8 @@ namespace shell {
 class AccessibilityBridge;
 }  // namespace shell
 
+@class FlutterPlatformViewSemanticsContainer;
+
 /**
  * A node in the iOS semantics tree.
  */
@@ -71,6 +73,11 @@ class AccessibilityBridge;
  */
 @property(nonatomic, strong) NSMutableArray<SemanticsObject*>* children;
 
+/**
+ * Used if this SemanticsObject is for a platform view.
+ */
+@property(strong, nonatomic) FlutterPlatformViewSemanticsContainer* platformViewSemanticsContainer;
+
 - (BOOL)nodeWillCauseLayoutChange:(const blink::SemanticsNode*)node;
 
 #pragma mark - Designated initializers
@@ -108,12 +115,31 @@ class AccessibilityBridge;
 @interface FlutterSemanticsObject : SemanticsObject
 @end
 
+/**
+ * Designated to act as an accessibility container of a platform view.
+ *
+ * This object does not take any accessibility actions on its own, nor has any accessibility
+ * label/value/trait/hint... on its own. The accessibility data will be handled by the platform
+ * view.
+ *
+ * See also:
+ * * `SemanticsObject` for the other type of semantics objects.
+ * * `FlutterSemanticsObject` for default implementation of `SemanticsObject`.
+ */
+@interface FlutterPlatformViewSemanticsContainer : UIAccessibilityElement
+
+- (instancetype)init __attribute__((unavailable("Use initWithAccessibilityContainer: instead")));
+
+@end
+
 namespace shell {
 class PlatformViewIOS;
 
 class AccessibilityBridge final {
  public:
-  AccessibilityBridge(UIView* view, PlatformViewIOS* platform_view);
+  AccessibilityBridge(UIView* view,
+                      PlatformViewIOS* platform_view,
+                      FlutterPlatformViewsController* platform_views_controller);
   ~AccessibilityBridge();
 
   void UpdateSemantics(blink::SemanticsNodeUpdates nodes,
@@ -129,6 +155,10 @@ class AccessibilityBridge final {
 
   fml::WeakPtr<AccessibilityBridge> GetWeakPtr();
 
+  FlutterPlatformViewsController* GetPlatformViewsController() const {
+    return platform_views_controller_;
+  };
+
   void clearState();
 
  private:
@@ -139,6 +169,7 @@ class AccessibilityBridge final {
 
   UIView* view_;
   PlatformViewIOS* platform_view_;
+  FlutterPlatformViewsController* platform_views_controller_;
   fml::scoped_nsobject<NSMutableDictionary<NSNumber*, SemanticsObject*>> objects_;
   fml::scoped_nsprotocol<FlutterBasicMessageChannel*> accessibility_channel_;
   fml::WeakPtrFactory<AccessibilityBridge> weak_factory_;

engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm

@@ -11,6 +11,7 @@
 #import <UIKit/UIKit.h>
 
 #include "flutter/fml/logging.h"
+#include "flutter/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews_Internal.h"
 #include "flutter/shell/platform/darwin/ios/platform_view_ios.h"
 
 namespace {
@@ -127,6 +128,7 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection(
   [_children release];
   _parent = nil;
   _container.get().semanticsObject = nil;
+  [_platformViewSemanticsContainer release];
   [super dealloc];
 }
 
@@ -152,6 +154,9 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection(
 }
 
 - (BOOL)hasChildren {
+  if (_node.IsPlatformViewNode()) {
+    return YES;
+  }
   return [self.children count] != 0;
 }
 
@@ -165,6 +170,7 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection(
   //  We enforce in the framework that no other useful semantics are merged with these nodes.
   if ([self node].HasFlag(blink::SemanticsFlags::kScopesRoute))
     return false;
+
   return ([self node].flags != 0 &&
           [self node].flags != static_cast<int32_t>(blink::SemanticsFlags::kIsHidden)) ||
          ![self node].label.empty() || ![self node].value.empty() || ![self node].hint.empty() ||
@@ -396,6 +402,25 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection(
 
 @end
 
+@implementation FlutterPlatformViewSemanticsContainer
+
+// Method declared as unavailable in the interface
+- (instancetype)init {
+  [self release];
+  [super doesNotRecognizeSelector:_cmd];
+  return nil;
+}
+
+- (instancetype)initWithAccessibilityContainer:(id)container {
+  FML_CHECK(container);
+  if (self = [super initWithAccessibilityContainer:container]) {
+    self.isAccessibilityElement = NO;
+  }
+  return self;
+}
+
+@end
+
 @implementation SemanticsObjectContainer {
   SemanticsObject* _semanticsObject;
   fml::WeakPtr<shell::AccessibilityBridge> _bridge;
@@ -426,7 +451,12 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection(
 #pragma mark - UIAccessibilityContainer overrides
 
 - (NSInteger)accessibilityElementCount {
-  return [[_semanticsObject children] count] + 1;
+  NSInteger count = [[_semanticsObject children] count] + 1;
+  // Need to create an additional child that acts as accessibility container for the platform view.
+  if (_semanticsObject.node.IsPlatformViewNode()) {
+    count++;
+  }
+  return count;
 }
 
 - (nullable id)accessibilityElementAtIndex:(NSInteger)index {
@@ -435,7 +465,16 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection(
   if (index == 0) {
     return _semanticsObject;
   }
+
+  // Return the additional child acts as a container of platform view. The
+  // platformViewSemanticsContainer was created and cached in the updateSemantics path.
+  if (_semanticsObject.node.IsPlatformViewNode() && index == [self accessibilityElementCount] - 1) {
+    FML_CHECK(_semanticsObject.platformViewSemanticsContainer != nil);
+    return _semanticsObject.platformViewSemanticsContainer;
+  }
+
   SemanticsObject* child = [_semanticsObject children][index - 1];
+
   if ([child hasChildren])
     return [child accessibilityContainer];
   return child;
@@ -444,6 +483,12 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection(
 - (NSInteger)indexOfAccessibilityElement:(id)element {
   if (element == _semanticsObject)
     return 0;
+
+  // FlutterPlatformViewSemanticsContainer is always the last element of its parent.
+  if ([element isKindOfClass:[FlutterPlatformViewSemanticsContainer class]]) {
+    return [self accessibilityElementCount] - 1;
+  }
+
   NSMutableArray<SemanticsObject*>* children = [_semanticsObject children];
   for (size_t i = 0; i < [children count]; i++) {
     SemanticsObject* child = children[i];
@@ -485,9 +530,12 @@ blink::SemanticsAction GetSemanticsActionForScrollDirection(
 
 namespace shell {
 
-AccessibilityBridge::AccessibilityBridge(UIView* view, PlatformViewIOS* platform_view)
+AccessibilityBridge::AccessibilityBridge(UIView* view,
+                                         PlatformViewIOS* platform_view,
+                                         FlutterPlatformViewsController* platform_views_controller)
     : view_(view),
       platform_view_(platform_view),
+      platform_views_controller_(platform_views_controller),
       objects_([[NSMutableDictionary alloc] init]),
       weak_factory_(this),
       previous_route_id_(0),
@@ -525,7 +573,7 @@ void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes,
     layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
     scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
     [object setSemanticsNode:&node];
-    const NSUInteger newChildCount = node.childrenInTraversalOrder.size();
+    NSUInteger newChildCount = node.childrenInTraversalOrder.size();
     NSMutableArray* newChildren =
         [[[NSMutableArray alloc] initWithCapacity:newChildCount] autorelease];
     for (NSUInteger i = 0; i < newChildCount; ++i) {
@@ -555,6 +603,20 @@ void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes,
       }
       object.accessibilityCustomActions = accessibilityCustomActions;
     }
+
+    if (object.node.IsPlatformViewNode()) {
+      shell::FlutterPlatformViewsController* controller = GetPlatformViewsController();
+      if (controller) {
+        object.platformViewSemanticsContainer = [[FlutterPlatformViewSemanticsContainer alloc]
+            initWithAccessibilityContainer:[object accessibilityContainer]];
+        UIView* platformView = [controller->GetPlatformViewByID(object.node.platformViewId) view];
+        if (platformView) {
+          object.platformViewSemanticsContainer.accessibilityElements = @[ platformView ];
+        }
+      }
+    } else if (object.platformViewSemanticsContainer) {
+      [object.platformViewSemanticsContainer release];
+    }
   }
 
   SemanticsObject* root = objects_.get()[@(kRootNodeId)];

engine/src/flutter/shell/platform/darwin/ios/platform_view_ios.mm

@@ -50,7 +50,8 @@ void PlatformViewIOS::SetOwnerViewController(fml::WeakPtr<FlutterViewController>
 
     if (accessibility_bridge_) {
       accessibility_bridge_.reset(
-          new AccessibilityBridge(static_cast<FlutterView*>(owner_controller_.get().view), this));
+          new AccessibilityBridge(static_cast<FlutterView*>(owner_controller_.get().view), this,
+                                  [owner_controller.get() platformViewsController]));
     }
     // Do not call `NotifyCreated()` here - let FlutterViewController take care
     // of that when its Viewport is sized.  If `NotifyCreated()` is called here,
@@ -96,7 +97,8 @@ void PlatformViewIOS::SetSemanticsEnabled(bool enabled) {
   }
   if (enabled && !accessibility_bridge_) {
     accessibility_bridge_ = std::make_unique<AccessibilityBridge>(
-        static_cast<FlutterView*>(owner_controller_.get().view), this);
+        static_cast<FlutterView*>(owner_controller_.get().view), this,
+        [owner_controller_.get() platformViewsController]);
   } else if (!enabled && accessibility_bridge_) {
     accessibility_bridge_.reset();
   }