[Embedder API] Add semantic string attributes (flutter/engine#44616)

Flutter's `SemanticNode`s use [`StringAttribute`](https://api.flutter.dev/flutter/dart-ui/StringAttribute-class.html)s to provide additional information on text values for assistive technologies. This exposes the string attributes on the embedder API so that embedders can apply string attributes to their semantics trees.

Addresses https://github.com/flutter/flutter/issues/119970
Part of https://github.com/flutter/flutter/issues/98948

Previous pull request: https://github.com/flutter/engine/pull/44553

[C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style

engine/src/flutter/lib/ui/semantics/string_attribute.h

@@ -20,6 +20,7 @@ using StringAttributes = std::vector<StringAttributePtr>;
 //  * engine/src/flutter/lib/ui/semantics.dart
 //  * engine/src/flutter/lib/web_ui/lib/semantics.dart
 //  * engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java
+//  * engine/src/flutter/shell/platform/embedder/embedder.h
 //  * engine/src/flutter/lib/web_ui/test/engine/semantics/semantics_api_test.dart
 //  * engine/src/flutter/testing/dart/semantics_test.dart
 

engine/src/flutter/shell/platform/embedder/embedder.h

@@ -25,9 +25,9 @@
 // - Function signatures (names, argument counts, argument order, and argument
 //   type) cannot change.
 // - The core behavior of existing functions cannot change.
-// - Instead of nesting structures by value within another structure, prefer
-//   nesting by pointer. This ensures that adding members to the nested struct
-//   does not break the ABI of the parent struct.
+// - Instead of nesting structures by value within another structure/union,
+//   prefer nesting by pointer. This ensures that adding members to the nested
+//   struct does not break the ABI of the parent struct/union.
 // - Instead of array of structures, prefer array of pointers to structures.
 //   This ensures that array indexing does not break if members are added
 //   to the structure.
@@ -1064,6 +1064,57 @@ typedef int64_t FlutterPlatformViewIdentifier;
 FLUTTER_EXPORT
 extern const int32_t kFlutterSemanticsNodeIdBatchEnd;
 
+// The enumeration of possible string attributes that affect how assistive
+// technologies announce a string.
+//
+// See dart:ui's implementers of the StringAttribute abstract class.
+typedef enum {
+  // Indicates the string should be announced character by character.
+  kSpellOut,
+  // Indicates the string should be announced using the specified locale.
+  kLocale,
+} FlutterStringAttributeType;
+
+// Indicates the assistive technology should announce out the string character
+// by character.
+//
+// See dart:ui's SpellOutStringAttribute.
+typedef struct {
+  /// The size of this struct. Must be sizeof(FlutterSpellOutStringAttribute).
+  size_t struct_size;
+} FlutterSpellOutStringAttribute;
+
+// Indicates the assistive technology should announce the string using the
+// specified locale.
+//
+// See dart:ui's LocaleStringAttribute.
+typedef struct {
+  /// The size of this struct. Must be sizeof(FlutterLocaleStringAttribute).
+  size_t struct_size;
+  // The locale of this attribute.
+  const char* locale;
+} FlutterLocaleStringAttribute;
+
+// Indicates how the assistive technology should treat the string.
+//
+// See dart:ui's StringAttribute.
+typedef struct {
+  /// The size of this struct. Must be sizeof(FlutterStringAttribute).
+  size_t struct_size;
+  // The position this attribute starts.
+  size_t start;
+  // The next position after the attribute ends.
+  size_t end;
+  /// The type of the attribute described by the subsequent union.
+  FlutterStringAttributeType type;
+  union {
+    // Indicates the string should be announced character by character.
+    const FlutterSpellOutStringAttribute* spell_out;
+    // Indicates the string should be announced using the specified locale.
+    const FlutterLocaleStringAttribute* locale;
+  };
+} FlutterStringAttribute;
+
 /// A node that represents some semantic data.
 ///
 /// The semantics tree is maintained during the semantics phase of the pipeline
@@ -1215,6 +1266,31 @@ typedef struct {
   FlutterPlatformViewIdentifier platform_view_id;
   /// A textual tooltip attached to the node.
   const char* tooltip;
+  // The number of string attributes associated with the `label`.
+  size_t label_attribute_count;
+  // Array of string attributes associated with the `label`.
+  // Has length `label_attribute_count`.
+  const FlutterStringAttribute** label_attributes;
+  // The number of string attributes associated with the `hint`.
+  size_t hint_attribute_count;
+  // Array of string attributes associated with the `hint`.
+  // Has length `hint_attribute_count`.
+  const FlutterStringAttribute** hint_attributes;
+  // The number of string attributes associated with the `value`.
+  size_t value_attribute_count;
+  // Array of string attributes associated with the `value`.
+  // Has length `value_attribute_count`.
+  const FlutterStringAttribute** value_attributes;
+  // The number of string attributes associated with the `increased_value`.
+  size_t increased_value_attribute_count;
+  // Array of string attributes associated with the `increased_value`.
+  // Has length `increased_value_attribute_count`.
+  const FlutterStringAttribute** increased_value_attributes;
+  // The number of string attributes associated with the `decreased_value`.
+  size_t decreased_value_attribute_count;
+  // Array of string attributes associated with the `decreased_value`.
+  // Has length `decreased_value_attribute_count`.
+  const FlutterStringAttribute** decreased_value_attributes;
 } FlutterSemanticsNode2;
 
 /// `FlutterSemanticsCustomAction` ID used as a sentinel to signal the end of a

engine/src/flutter/shell/platform/embedder/embedder_semantics_update.cc

@@ -131,6 +131,14 @@ void EmbedderSemanticsUpdate2::AddNode(const SemanticsNode& node) {
       transform.get(SkMatrix::kMPersp0), transform.get(SkMatrix::kMPersp1),
       transform.get(SkMatrix::kMPersp2)};
 
+  auto label_attributes = CreateStringAttributes(node.labelAttributes);
+  auto hint_attributes = CreateStringAttributes(node.hintAttributes);
+  auto value_attributes = CreateStringAttributes(node.valueAttributes);
+  auto increased_value_attributes =
+      CreateStringAttributes(node.increasedValueAttributes);
+  auto decreased_value_attributes =
+      CreateStringAttributes(node.decreasedValueAttributes);
+
   nodes_.push_back({
       sizeof(FlutterSemanticsNode2),
       node.id,
@@ -161,6 +169,16 @@ void EmbedderSemanticsUpdate2::AddNode(const SemanticsNode& node) {
       node.customAccessibilityActions.data(),
       node.platformViewId,
       node.tooltip.c_str(),
+      label_attributes.count,
+      label_attributes.attributes,
+      hint_attributes.count,
+      hint_attributes.attributes,
+      value_attributes.count,
+      value_attributes.attributes,
+      increased_value_attributes.count,
+      increased_value_attributes.attributes,
+      decreased_value_attributes.count,
+      decreased_value_attributes.attributes,
   });
 }
 
@@ -175,4 +193,69 @@ void EmbedderSemanticsUpdate2::AddAction(
   });
 }
 
+EmbedderSemanticsUpdate2::EmbedderStringAttributes
+EmbedderSemanticsUpdate2::CreateStringAttributes(
+    const StringAttributes& attributes) {
+  // Minimize allocations if attributes are empty.
+  if (attributes.empty()) {
+    return {.count = 0, .attributes = nullptr};
+  }
+
+  // Translate the engine attributes to embedder attributes.
+  // The result vector's data is returned by this method.
+  // The result vector will be owned by |node_string_attributes_|
+  // so that the embedder attributes are cleaned up at the end of the
+  // semantics update callback when when the |EmbedderSemanticsUpdate2|
+  // is destroyed.
+  auto result = std::make_unique<std::vector<const FlutterStringAttribute*>>();
+  result->reserve(attributes.size());
+
+  for (const auto& attribute : attributes) {
+    auto embedder_attribute = std::make_unique<FlutterStringAttribute>();
+    embedder_attribute->struct_size = sizeof(FlutterStringAttribute);
+    embedder_attribute->start = attribute->start;
+    embedder_attribute->end = attribute->end;
+
+    switch (attribute->type) {
+      case StringAttributeType::kLocale: {
+        std::shared_ptr<flutter::LocaleStringAttribute> locale_attribute =
+            std::static_pointer_cast<flutter::LocaleStringAttribute>(attribute);
+
+        auto embedder_locale = std::make_unique<FlutterLocaleStringAttribute>();
+        embedder_locale->struct_size = sizeof(FlutterLocaleStringAttribute);
+        embedder_locale->locale = locale_attribute->locale.c_str();
+        locale_attributes_.push_back(std::move(embedder_locale));
+
+        embedder_attribute->type = FlutterStringAttributeType::kLocale;
+        embedder_attribute->locale = locale_attributes_.back().get();
+        break;
+      }
+      case flutter::StringAttributeType::kSpellOut: {
+        // All spell out attributes are identical and share a lazily created
+        // instance.
+        if (!spell_out_attribute_) {
+          auto spell_out_attribute_ =
+              std::make_unique<FlutterSpellOutStringAttribute>();
+          spell_out_attribute_->struct_size =
+              sizeof(FlutterSpellOutStringAttribute);
+        }
+
+        embedder_attribute->type = FlutterStringAttributeType::kSpellOut;
+        embedder_attribute->spell_out = spell_out_attribute_.get();
+        break;
+      }
+    }
+
+    string_attributes_.push_back(std::move(embedder_attribute));
+    result->push_back(string_attributes_.back().get());
+  }
+
+  node_string_attributes_.push_back(std::move(result));
+
+  return {
+      .count = node_string_attributes_.back()->size(),
+      .attributes = node_string_attributes_.back()->data(),
+  };
+}
+
 }  // namespace flutter

engine/src/flutter/shell/platform/embedder/embedder_semantics_update.h

@@ -40,6 +40,11 @@ class EmbedderSemanticsUpdate {
 };
 
 // A semantic update, used by the embedder API's v3 semantic update callback.
+//
+// This holds temporary embedder-specific objects that are translated from
+// the engine's internal representation and passed back to the semantics
+// update callback. Once the callback finishes, this object is destroyed
+// and the temporary embedder-specific objects are automatically cleaned up.
 class EmbedderSemanticsUpdate2 {
  public:
   EmbedderSemanticsUpdate2(const SemanticsNodeUpdates& nodes,
@@ -52,12 +57,22 @@ class EmbedderSemanticsUpdate2 {
   FlutterSemanticsUpdate2* get() { return &update_; }
 
  private:
+  // These fields hold temporary embedder-specific objects that
+  // must remain valid for the duration of the semantics update callback.
+  // They are automatically cleaned up when |EmbedderSemanticsUpdate2| is
+  // destroyed.
   FlutterSemanticsUpdate2 update_;
   std::vector<FlutterSemanticsNode2> nodes_;
   std::vector<FlutterSemanticsNode2*> node_pointers_;
   std::vector<FlutterSemanticsCustomAction2> actions_;
   std::vector<FlutterSemanticsCustomAction2*> action_pointers_;
 
+  std::vector<std::unique_ptr<std::vector<const FlutterStringAttribute*>>>
+      node_string_attributes_;
+  std::vector<std::unique_ptr<FlutterStringAttribute>> string_attributes_;
+  std::vector<std::unique_ptr<FlutterLocaleStringAttribute>> locale_attributes_;
+  std::unique_ptr<FlutterSpellOutStringAttribute> spell_out_attribute_;
+
   // Translates engine semantic nodes to embedder semantic nodes.
   void AddNode(const SemanticsNode& node);
 
@@ -65,6 +80,18 @@ class EmbedderSemanticsUpdate2 {
   // actions.
   void AddAction(const CustomAccessibilityAction& action);
 
+  // A helper struct for |CreateStringAttributes|.
+  struct EmbedderStringAttributes {
+    // The number of string attribute pointers in |attributes|.
+    size_t count;
+    // An array of string attribute pointers.
+    const FlutterStringAttribute** attributes;
+  };
+
+  // Translates engine string attributes to embedder string attributes.
+  EmbedderStringAttributes CreateStringAttributes(
+      const StringAttributes& attribute);
+
   FML_DISALLOW_COPY_AND_ASSIGN(EmbedderSemanticsUpdate2);
 };
 

engine/src/flutter/shell/platform/embedder/fixtures/main.dart

@@ -290,6 +290,82 @@ void a11y_main() async {
   notifySemanticsEnabled(PlatformDispatcher.instance.semanticsEnabled);
 }
 
+@pragma('vm:entry-point')
+void a11y_string_attributes() async {
+  // 1: Wait until semantics are enabled.
+  if (!PlatformDispatcher.instance.semanticsEnabled) {
+    await semanticsChanged;
+  }
+
+  // 2: Update semantics with string attributes.
+  final SemanticsUpdateBuilder builder = SemanticsUpdateBuilder()
+    ..updateNode(
+      id: 42,
+      label: 'What is the meaning of life?',
+      labelAttributes: <StringAttribute>[
+        LocaleStringAttribute(
+          range: TextRange(start: 0, end: 'What is the meaning of life?'.length),
+          locale: Locale('en'),
+        ),
+        SpellOutStringAttribute(
+          range: TextRange(start: 0, end: 1),
+        ),
+      ],
+      rect: Rect.fromLTRB(0.0, 0.0, 10.0, 10.0),
+      transform: kTestTransform,
+      childrenInTraversalOrder: Int32List.fromList(<int>[84, 96]),
+      childrenInHitTestOrder: Int32List.fromList(<int>[96, 84]),
+      actions: 0,
+      flags: 0,
+      maxValueLength: 0,
+      currentValueLength: 0,
+      textSelectionBase: 0,
+      textSelectionExtent: 0,
+      platformViewId: 0,
+      scrollChildren: 0,
+      scrollIndex: 0,
+      scrollPosition: 0.0,
+      scrollExtentMax: 0.0,
+      scrollExtentMin: 0.0,
+      elevation: 0.0,
+      thickness: 0.0,
+      hint: "It's a number",
+      hintAttributes: <StringAttribute>[
+        LocaleStringAttribute(
+          range: TextRange(start: 0, end: 1),
+          locale: Locale('en'),
+        ),
+        LocaleStringAttribute(
+          range: TextRange(start: 2, end: 3),
+          locale: Locale('fr'),
+        ),
+      ],
+      value: '42',
+      valueAttributes: <StringAttribute>[
+        LocaleStringAttribute(
+          range: TextRange(start: 0, end: '42'.length),
+          locale: Locale('en', 'US'),
+        ),
+      ],
+      increasedValue: '43',
+      increasedValueAttributes: <StringAttribute>[
+        SpellOutStringAttribute(
+          range: TextRange(start: 0, end: 1),
+        ),
+        SpellOutStringAttribute(
+          range: TextRange(start: 1, end: 2),
+        ),
+      ],
+      decreasedValue: '41',
+      decreasedValueAttributes: <StringAttribute>[],
+      tooltip: 'tooltip',
+      additionalActions: Int32List(0),
+    );
+
+  PlatformDispatcher.instance.views.first.updateSemantics(builder.build());
+  signalNativeTest();
+}
+
 @pragma('vm:entry-point')
 void platform_messages_response() {
   PlatformDispatcher.instance.onPlatformMessage =

engine/src/flutter/shell/platform/embedder/tests/embedder_a11y_unittests.cc

@@ -272,6 +272,126 @@ TEST_F(EmbedderA11yTest, A11yTreeIsConsistentUsingV3Callbacks) {
 #endif  // OS_FUCHSIA
 }
 
+TEST_F(EmbedderA11yTest, A11yStringAttributes) {
+#if defined(OS_FUCHSIA)
+  GTEST_SKIP() << "This test crashes on Fuchsia. https://fxbug.dev/87493 ";
+#else
+
+  auto& context = GetEmbedderContext(EmbedderTestContextType::kSoftwareContext);
+
+  fml::AutoResetWaitableEvent signal_native_latch;
+
+  // Called by the Dart text fixture on the UI thread to signal that the C++
+  // unittest should resume.
+  context.AddNativeCallback(
+      "SignalNativeTest",
+      CREATE_NATIVE_ENTRY(([&signal_native_latch](Dart_NativeArguments) {
+        signal_native_latch.Signal();
+      })));
+
+  fml::AutoResetWaitableEvent semantics_update_latch;
+  context.SetSemanticsUpdateCallback2(
+      [&](const FlutterSemanticsUpdate2* update) {
+        ASSERT_EQ(update->node_count, size_t(1));
+        ASSERT_EQ(update->custom_action_count, size_t(0));
+
+        auto node = update->nodes[0];
+
+        // Verify label
+        {
+          ASSERT_EQ(std::string(node->label), "What is the meaning of life?");
+          ASSERT_EQ(node->label_attribute_count, size_t(2));
+
+          ASSERT_EQ(node->label_attributes[0]->start, size_t(0));
+          ASSERT_EQ(node->label_attributes[0]->end, size_t(28));
+          ASSERT_EQ(node->label_attributes[0]->type,
+                    FlutterStringAttributeType::kLocale);
+          ASSERT_EQ(std::string(node->label_attributes[0]->locale->locale),
+                    "en");
+
+          ASSERT_EQ(node->label_attributes[1]->start, size_t(0));
+          ASSERT_EQ(node->label_attributes[1]->end, size_t(1));
+          ASSERT_EQ(node->label_attributes[1]->type,
+                    FlutterStringAttributeType::kSpellOut);
+        }
+
+        // Verify hint
+        {
+          ASSERT_EQ(std::string(node->hint), "It's a number");
+          ASSERT_EQ(node->hint_attribute_count, size_t(2));
+
+          ASSERT_EQ(node->hint_attributes[0]->start, size_t(0));
+          ASSERT_EQ(node->hint_attributes[0]->end, size_t(1));
+          ASSERT_EQ(node->hint_attributes[0]->type,
+                    FlutterStringAttributeType::kLocale);
+          ASSERT_EQ(std::string(node->hint_attributes[0]->locale->locale),
+                    "en");
+
+          ASSERT_EQ(node->hint_attributes[1]->start, size_t(2));
+          ASSERT_EQ(node->hint_attributes[1]->end, size_t(3));
+          ASSERT_EQ(node->hint_attributes[1]->type,
+                    FlutterStringAttributeType::kLocale);
+          ASSERT_EQ(std::string(node->hint_attributes[1]->locale->locale),
+                    "fr");
+        }
+
+        // Verify value
+        {
+          ASSERT_EQ(std::string(node->value), "42");
+          ASSERT_EQ(node->value_attribute_count, size_t(1));
+
+          ASSERT_EQ(node->value_attributes[0]->start, size_t(0));
+          ASSERT_EQ(node->value_attributes[0]->end, size_t(2));
+          ASSERT_EQ(node->value_attributes[0]->type,
+                    FlutterStringAttributeType::kLocale);
+          ASSERT_EQ(std::string(node->value_attributes[0]->locale->locale),
+                    "en-US");
+        }
+
+        // Verify increased value
+        {
+          ASSERT_EQ(std::string(node->increased_value), "43");
+          ASSERT_EQ(node->increased_value_attribute_count, size_t(2));
+
+          ASSERT_EQ(node->increased_value_attributes[0]->start, size_t(0));
+          ASSERT_EQ(node->increased_value_attributes[0]->end, size_t(1));
+          ASSERT_EQ(node->increased_value_attributes[0]->type,
+                    FlutterStringAttributeType::kSpellOut);
+
+          ASSERT_EQ(node->increased_value_attributes[1]->start, size_t(1));
+          ASSERT_EQ(node->increased_value_attributes[1]->end, size_t(2));
+          ASSERT_EQ(node->increased_value_attributes[1]->type,
+                    FlutterStringAttributeType::kSpellOut);
+        }
+
+        // Verify decreased value
+        {
+          ASSERT_EQ(std::string(node->decreased_value), "41");
+          ASSERT_EQ(node->decreased_value_attribute_count, size_t(0));
+          ASSERT_EQ(node->decreased_value_attributes, nullptr);
+        }
+
+        semantics_update_latch.Signal();
+      });
+
+  EmbedderConfigBuilder builder(context);
+  builder.SetSoftwareRendererConfig();
+  builder.SetDartEntrypoint("a11y_string_attributes");
+
+  auto engine = builder.LaunchEngine();
+  ASSERT_TRUE(engine.is_valid());
+
+  // 1: Enable semantics.
+  auto result = FlutterEngineUpdateSemanticsEnabled(engine.get(), true);
+  ASSERT_EQ(result, FlutterEngineResult::kSuccess);
+
+  // 2: Wait for semantics update callback on platform (current) thread.
+  signal_native_latch.Wait();
+  fml::MessageLoop::GetCurrent().RunExpiredTasksNow();
+  semantics_update_latch.Wait();
+#endif  // OS_FUCHSIA
+}
+
 TEST_F(EmbedderA11yTest, A11yTreeIsConsistentUsingV2Callbacks) {
 #if defined(OS_FUCHSIA)
   GTEST_SKIP() << "This test crashes on Fuchsia. https://fxbug.dev/87493 ";