diff --git a/engine/src/flutter/lib/ui/semantics/string_attribute.h b/engine/src/flutter/lib/ui/semantics/string_attribute.h index b9ee3bc958..1e2dce57e6 100644 --- a/engine/src/flutter/lib/ui/semantics/string_attribute.h +++ b/engine/src/flutter/lib/ui/semantics/string_attribute.h @@ -20,6 +20,7 @@ using StringAttributes = std::vector; // * engine/src/flutter/lib/ui/semantics.dart // * engine/src/flutter/lib/web_ui/lib/semantics.dart // * engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java +// * engine/src/flutter/shell/platform/embedder/embedder.h // * engine/src/flutter/lib/web_ui/test/engine/semantics/semantics_api_test.dart // * engine/src/flutter/testing/dart/semantics_test.dart diff --git a/engine/src/flutter/shell/platform/embedder/embedder.h b/engine/src/flutter/shell/platform/embedder/embedder.h index a1e9036b68..730a45d184 100644 --- a/engine/src/flutter/shell/platform/embedder/embedder.h +++ b/engine/src/flutter/shell/platform/embedder/embedder.h @@ -25,9 +25,9 @@ // - Function signatures (names, argument counts, argument order, and argument // type) cannot change. // - The core behavior of existing functions cannot change. -// - Instead of nesting structures by value within another structure, prefer -// nesting by pointer. This ensures that adding members to the nested struct -// does not break the ABI of the parent struct. +// - Instead of nesting structures by value within another structure/union, +// prefer nesting by pointer. This ensures that adding members to the nested +// struct does not break the ABI of the parent struct/union. // - Instead of array of structures, prefer array of pointers to structures. // This ensures that array indexing does not break if members are added // to the structure. @@ -1064,6 +1064,57 @@ typedef int64_t FlutterPlatformViewIdentifier; FLUTTER_EXPORT extern const int32_t kFlutterSemanticsNodeIdBatchEnd; +// The enumeration of possible string attributes that affect how assistive +// technologies announce a string. +// +// See dart:ui's implementers of the StringAttribute abstract class. +typedef enum { + // Indicates the string should be announced character by character. + kSpellOut, + // Indicates the string should be announced using the specified locale. + kLocale, +} FlutterStringAttributeType; + +// Indicates the assistive technology should announce out the string character +// by character. +// +// See dart:ui's SpellOutStringAttribute. +typedef struct { + /// The size of this struct. Must be sizeof(FlutterSpellOutStringAttribute). + size_t struct_size; +} FlutterSpellOutStringAttribute; + +// Indicates the assistive technology should announce the string using the +// specified locale. +// +// See dart:ui's LocaleStringAttribute. +typedef struct { + /// The size of this struct. Must be sizeof(FlutterLocaleStringAttribute). + size_t struct_size; + // The locale of this attribute. + const char* locale; +} FlutterLocaleStringAttribute; + +// Indicates how the assistive technology should treat the string. +// +// See dart:ui's StringAttribute. +typedef struct { + /// The size of this struct. Must be sizeof(FlutterStringAttribute). + size_t struct_size; + // The position this attribute starts. + size_t start; + // The next position after the attribute ends. + size_t end; + /// The type of the attribute described by the subsequent union. + FlutterStringAttributeType type; + union { + // Indicates the string should be announced character by character. + const FlutterSpellOutStringAttribute* spell_out; + // Indicates the string should be announced using the specified locale. + const FlutterLocaleStringAttribute* locale; + }; +} FlutterStringAttribute; + /// A node that represents some semantic data. /// /// The semantics tree is maintained during the semantics phase of the pipeline @@ -1215,6 +1266,31 @@ typedef struct { FlutterPlatformViewIdentifier platform_view_id; /// A textual tooltip attached to the node. const char* tooltip; + // The number of string attributes associated with the `label`. + size_t label_attribute_count; + // Array of string attributes associated with the `label`. + // Has length `label_attribute_count`. + const FlutterStringAttribute** label_attributes; + // The number of string attributes associated with the `hint`. + size_t hint_attribute_count; + // Array of string attributes associated with the `hint`. + // Has length `hint_attribute_count`. + const FlutterStringAttribute** hint_attributes; + // The number of string attributes associated with the `value`. + size_t value_attribute_count; + // Array of string attributes associated with the `value`. + // Has length `value_attribute_count`. + const FlutterStringAttribute** value_attributes; + // The number of string attributes associated with the `increased_value`. + size_t increased_value_attribute_count; + // Array of string attributes associated with the `increased_value`. + // Has length `increased_value_attribute_count`. + const FlutterStringAttribute** increased_value_attributes; + // The number of string attributes associated with the `decreased_value`. + size_t decreased_value_attribute_count; + // Array of string attributes associated with the `decreased_value`. + // Has length `decreased_value_attribute_count`. + const FlutterStringAttribute** decreased_value_attributes; } FlutterSemanticsNode2; /// `FlutterSemanticsCustomAction` ID used as a sentinel to signal the end of a diff --git a/engine/src/flutter/shell/platform/embedder/embedder_semantics_update.cc b/engine/src/flutter/shell/platform/embedder/embedder_semantics_update.cc index a2abe3ac17..1670d7c734 100644 --- a/engine/src/flutter/shell/platform/embedder/embedder_semantics_update.cc +++ b/engine/src/flutter/shell/platform/embedder/embedder_semantics_update.cc @@ -131,6 +131,14 @@ void EmbedderSemanticsUpdate2::AddNode(const SemanticsNode& node) { transform.get(SkMatrix::kMPersp0), transform.get(SkMatrix::kMPersp1), transform.get(SkMatrix::kMPersp2)}; + auto label_attributes = CreateStringAttributes(node.labelAttributes); + auto hint_attributes = CreateStringAttributes(node.hintAttributes); + auto value_attributes = CreateStringAttributes(node.valueAttributes); + auto increased_value_attributes = + CreateStringAttributes(node.increasedValueAttributes); + auto decreased_value_attributes = + CreateStringAttributes(node.decreasedValueAttributes); + nodes_.push_back({ sizeof(FlutterSemanticsNode2), node.id, @@ -161,6 +169,16 @@ void EmbedderSemanticsUpdate2::AddNode(const SemanticsNode& node) { node.customAccessibilityActions.data(), node.platformViewId, node.tooltip.c_str(), + label_attributes.count, + label_attributes.attributes, + hint_attributes.count, + hint_attributes.attributes, + value_attributes.count, + value_attributes.attributes, + increased_value_attributes.count, + increased_value_attributes.attributes, + decreased_value_attributes.count, + decreased_value_attributes.attributes, }); } @@ -175,4 +193,69 @@ void EmbedderSemanticsUpdate2::AddAction( }); } +EmbedderSemanticsUpdate2::EmbedderStringAttributes +EmbedderSemanticsUpdate2::CreateStringAttributes( + const StringAttributes& attributes) { + // Minimize allocations if attributes are empty. + if (attributes.empty()) { + return {.count = 0, .attributes = nullptr}; + } + + // Translate the engine attributes to embedder attributes. + // The result vector's data is returned by this method. + // The result vector will be owned by |node_string_attributes_| + // so that the embedder attributes are cleaned up at the end of the + // semantics update callback when when the |EmbedderSemanticsUpdate2| + // is destroyed. + auto result = std::make_unique>(); + result->reserve(attributes.size()); + + for (const auto& attribute : attributes) { + auto embedder_attribute = std::make_unique(); + embedder_attribute->struct_size = sizeof(FlutterStringAttribute); + embedder_attribute->start = attribute->start; + embedder_attribute->end = attribute->end; + + switch (attribute->type) { + case StringAttributeType::kLocale: { + std::shared_ptr locale_attribute = + std::static_pointer_cast(attribute); + + auto embedder_locale = std::make_unique(); + embedder_locale->struct_size = sizeof(FlutterLocaleStringAttribute); + embedder_locale->locale = locale_attribute->locale.c_str(); + locale_attributes_.push_back(std::move(embedder_locale)); + + embedder_attribute->type = FlutterStringAttributeType::kLocale; + embedder_attribute->locale = locale_attributes_.back().get(); + break; + } + case flutter::StringAttributeType::kSpellOut: { + // All spell out attributes are identical and share a lazily created + // instance. + if (!spell_out_attribute_) { + auto spell_out_attribute_ = + std::make_unique(); + spell_out_attribute_->struct_size = + sizeof(FlutterSpellOutStringAttribute); + } + + embedder_attribute->type = FlutterStringAttributeType::kSpellOut; + embedder_attribute->spell_out = spell_out_attribute_.get(); + break; + } + } + + string_attributes_.push_back(std::move(embedder_attribute)); + result->push_back(string_attributes_.back().get()); + } + + node_string_attributes_.push_back(std::move(result)); + + return { + .count = node_string_attributes_.back()->size(), + .attributes = node_string_attributes_.back()->data(), + }; +} + } // namespace flutter diff --git a/engine/src/flutter/shell/platform/embedder/embedder_semantics_update.h b/engine/src/flutter/shell/platform/embedder/embedder_semantics_update.h index 1816489ad2..1e628e41ad 100644 --- a/engine/src/flutter/shell/platform/embedder/embedder_semantics_update.h +++ b/engine/src/flutter/shell/platform/embedder/embedder_semantics_update.h @@ -40,6 +40,11 @@ class EmbedderSemanticsUpdate { }; // A semantic update, used by the embedder API's v3 semantic update callback. +// +// This holds temporary embedder-specific objects that are translated from +// the engine's internal representation and passed back to the semantics +// update callback. Once the callback finishes, this object is destroyed +// and the temporary embedder-specific objects are automatically cleaned up. class EmbedderSemanticsUpdate2 { public: EmbedderSemanticsUpdate2(const SemanticsNodeUpdates& nodes, @@ -52,12 +57,22 @@ class EmbedderSemanticsUpdate2 { FlutterSemanticsUpdate2* get() { return &update_; } private: + // These fields hold temporary embedder-specific objects that + // must remain valid for the duration of the semantics update callback. + // They are automatically cleaned up when |EmbedderSemanticsUpdate2| is + // destroyed. FlutterSemanticsUpdate2 update_; std::vector nodes_; std::vector node_pointers_; std::vector actions_; std::vector action_pointers_; + std::vector>> + node_string_attributes_; + std::vector> string_attributes_; + std::vector> locale_attributes_; + std::unique_ptr spell_out_attribute_; + // Translates engine semantic nodes to embedder semantic nodes. void AddNode(const SemanticsNode& node); @@ -65,6 +80,18 @@ class EmbedderSemanticsUpdate2 { // actions. void AddAction(const CustomAccessibilityAction& action); + // A helper struct for |CreateStringAttributes|. + struct EmbedderStringAttributes { + // The number of string attribute pointers in |attributes|. + size_t count; + // An array of string attribute pointers. + const FlutterStringAttribute** attributes; + }; + + // Translates engine string attributes to embedder string attributes. + EmbedderStringAttributes CreateStringAttributes( + const StringAttributes& attribute); + FML_DISALLOW_COPY_AND_ASSIGN(EmbedderSemanticsUpdate2); }; diff --git a/engine/src/flutter/shell/platform/embedder/fixtures/main.dart b/engine/src/flutter/shell/platform/embedder/fixtures/main.dart index 183da88987..5015c1af8c 100644 --- a/engine/src/flutter/shell/platform/embedder/fixtures/main.dart +++ b/engine/src/flutter/shell/platform/embedder/fixtures/main.dart @@ -290,6 +290,82 @@ void a11y_main() async { notifySemanticsEnabled(PlatformDispatcher.instance.semanticsEnabled); } +@pragma('vm:entry-point') +void a11y_string_attributes() async { + // 1: Wait until semantics are enabled. + if (!PlatformDispatcher.instance.semanticsEnabled) { + await semanticsChanged; + } + + // 2: Update semantics with string attributes. + final SemanticsUpdateBuilder builder = SemanticsUpdateBuilder() + ..updateNode( + id: 42, + label: 'What is the meaning of life?', + labelAttributes: [ + LocaleStringAttribute( + range: TextRange(start: 0, end: 'What is the meaning of life?'.length), + locale: Locale('en'), + ), + SpellOutStringAttribute( + range: TextRange(start: 0, end: 1), + ), + ], + rect: Rect.fromLTRB(0.0, 0.0, 10.0, 10.0), + transform: kTestTransform, + childrenInTraversalOrder: Int32List.fromList([84, 96]), + childrenInHitTestOrder: Int32List.fromList([96, 84]), + actions: 0, + flags: 0, + maxValueLength: 0, + currentValueLength: 0, + textSelectionBase: 0, + textSelectionExtent: 0, + platformViewId: 0, + scrollChildren: 0, + scrollIndex: 0, + scrollPosition: 0.0, + scrollExtentMax: 0.0, + scrollExtentMin: 0.0, + elevation: 0.0, + thickness: 0.0, + hint: "It's a number", + hintAttributes: [ + LocaleStringAttribute( + range: TextRange(start: 0, end: 1), + locale: Locale('en'), + ), + LocaleStringAttribute( + range: TextRange(start: 2, end: 3), + locale: Locale('fr'), + ), + ], + value: '42', + valueAttributes: [ + LocaleStringAttribute( + range: TextRange(start: 0, end: '42'.length), + locale: Locale('en', 'US'), + ), + ], + increasedValue: '43', + increasedValueAttributes: [ + SpellOutStringAttribute( + range: TextRange(start: 0, end: 1), + ), + SpellOutStringAttribute( + range: TextRange(start: 1, end: 2), + ), + ], + decreasedValue: '41', + decreasedValueAttributes: [], + tooltip: 'tooltip', + additionalActions: Int32List(0), + ); + + PlatformDispatcher.instance.views.first.updateSemantics(builder.build()); + signalNativeTest(); +} + @pragma('vm:entry-point') void platform_messages_response() { PlatformDispatcher.instance.onPlatformMessage = diff --git a/engine/src/flutter/shell/platform/embedder/tests/embedder_a11y_unittests.cc b/engine/src/flutter/shell/platform/embedder/tests/embedder_a11y_unittests.cc index 181a523a07..01cfff7cac 100644 --- a/engine/src/flutter/shell/platform/embedder/tests/embedder_a11y_unittests.cc +++ b/engine/src/flutter/shell/platform/embedder/tests/embedder_a11y_unittests.cc @@ -272,6 +272,126 @@ TEST_F(EmbedderA11yTest, A11yTreeIsConsistentUsingV3Callbacks) { #endif // OS_FUCHSIA } +TEST_F(EmbedderA11yTest, A11yStringAttributes) { +#if defined(OS_FUCHSIA) + GTEST_SKIP() << "This test crashes on Fuchsia. https://fxbug.dev/87493 "; +#else + + auto& context = GetEmbedderContext(EmbedderTestContextType::kSoftwareContext); + + fml::AutoResetWaitableEvent signal_native_latch; + + // Called by the Dart text fixture on the UI thread to signal that the C++ + // unittest should resume. + context.AddNativeCallback( + "SignalNativeTest", + CREATE_NATIVE_ENTRY(([&signal_native_latch](Dart_NativeArguments) { + signal_native_latch.Signal(); + }))); + + fml::AutoResetWaitableEvent semantics_update_latch; + context.SetSemanticsUpdateCallback2( + [&](const FlutterSemanticsUpdate2* update) { + ASSERT_EQ(update->node_count, size_t(1)); + ASSERT_EQ(update->custom_action_count, size_t(0)); + + auto node = update->nodes[0]; + + // Verify label + { + ASSERT_EQ(std::string(node->label), "What is the meaning of life?"); + ASSERT_EQ(node->label_attribute_count, size_t(2)); + + ASSERT_EQ(node->label_attributes[0]->start, size_t(0)); + ASSERT_EQ(node->label_attributes[0]->end, size_t(28)); + ASSERT_EQ(node->label_attributes[0]->type, + FlutterStringAttributeType::kLocale); + ASSERT_EQ(std::string(node->label_attributes[0]->locale->locale), + "en"); + + ASSERT_EQ(node->label_attributes[1]->start, size_t(0)); + ASSERT_EQ(node->label_attributes[1]->end, size_t(1)); + ASSERT_EQ(node->label_attributes[1]->type, + FlutterStringAttributeType::kSpellOut); + } + + // Verify hint + { + ASSERT_EQ(std::string(node->hint), "It's a number"); + ASSERT_EQ(node->hint_attribute_count, size_t(2)); + + ASSERT_EQ(node->hint_attributes[0]->start, size_t(0)); + ASSERT_EQ(node->hint_attributes[0]->end, size_t(1)); + ASSERT_EQ(node->hint_attributes[0]->type, + FlutterStringAttributeType::kLocale); + ASSERT_EQ(std::string(node->hint_attributes[0]->locale->locale), + "en"); + + ASSERT_EQ(node->hint_attributes[1]->start, size_t(2)); + ASSERT_EQ(node->hint_attributes[1]->end, size_t(3)); + ASSERT_EQ(node->hint_attributes[1]->type, + FlutterStringAttributeType::kLocale); + ASSERT_EQ(std::string(node->hint_attributes[1]->locale->locale), + "fr"); + } + + // Verify value + { + ASSERT_EQ(std::string(node->value), "42"); + ASSERT_EQ(node->value_attribute_count, size_t(1)); + + ASSERT_EQ(node->value_attributes[0]->start, size_t(0)); + ASSERT_EQ(node->value_attributes[0]->end, size_t(2)); + ASSERT_EQ(node->value_attributes[0]->type, + FlutterStringAttributeType::kLocale); + ASSERT_EQ(std::string(node->value_attributes[0]->locale->locale), + "en-US"); + } + + // Verify increased value + { + ASSERT_EQ(std::string(node->increased_value), "43"); + ASSERT_EQ(node->increased_value_attribute_count, size_t(2)); + + ASSERT_EQ(node->increased_value_attributes[0]->start, size_t(0)); + ASSERT_EQ(node->increased_value_attributes[0]->end, size_t(1)); + ASSERT_EQ(node->increased_value_attributes[0]->type, + FlutterStringAttributeType::kSpellOut); + + ASSERT_EQ(node->increased_value_attributes[1]->start, size_t(1)); + ASSERT_EQ(node->increased_value_attributes[1]->end, size_t(2)); + ASSERT_EQ(node->increased_value_attributes[1]->type, + FlutterStringAttributeType::kSpellOut); + } + + // Verify decreased value + { + ASSERT_EQ(std::string(node->decreased_value), "41"); + ASSERT_EQ(node->decreased_value_attribute_count, size_t(0)); + ASSERT_EQ(node->decreased_value_attributes, nullptr); + } + + semantics_update_latch.Signal(); + }); + + EmbedderConfigBuilder builder(context); + builder.SetSoftwareRendererConfig(); + builder.SetDartEntrypoint("a11y_string_attributes"); + + auto engine = builder.LaunchEngine(); + ASSERT_TRUE(engine.is_valid()); + + // 1: Enable semantics. + auto result = FlutterEngineUpdateSemanticsEnabled(engine.get(), true); + ASSERT_EQ(result, FlutterEngineResult::kSuccess); + + // 2: Wait for semantics update callback on platform (current) thread. + signal_native_latch.Wait(); + fml::MessageLoop::GetCurrent().RunExpiredTasksNow(); + semantics_update_latch.Wait(); +#endif // OS_FUCHSIA +} + TEST_F(EmbedderA11yTest, A11yTreeIsConsistentUsingV2Callbacks) { #if defined(OS_FUCHSIA) GTEST_SKIP() << "This test crashes on Fuchsia. https://fxbug.dev/87493 ";