Support for accessibility label and hint (flutter/engine#4264)

* Support for accessibility label and hint

* review comments

engine/src/flutter/lib/ui/semantics.dart

@@ -214,8 +214,10 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
   /// asynchronously, the [Window.onSemanticsAction] callback might be called
   /// with an action that is no longer possible.
   ///
-  /// The `label` is a string that describes this node. Its reading direction is
-  /// given by `textDirection`.
+  /// The `label` is a string that describes this node. The `value` property
+  /// describes the current value of the node as a string. The `hint` string
+  /// describes what result an action performed on this node has. The reading
+  /// direction of all these strings is given by `textDirection`.
   ///
   /// The `rect` is the region occupied by this node in its own coordinate
   /// system.
@@ -228,6 +230,8 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
     int actions,
     Rect rect,
     String label,
+    String hint,
+    String value,
     TextDirection textDirection,
     Float64List transform,
     Int32List children
@@ -242,6 +246,8 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
                 rect.right,
                 rect.bottom,
                 label,
+                hint,
+                value,
                 textDirection != null ? textDirection.index + 1 : 0,
                 transform,
                 children);
@@ -255,6 +261,8 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
     double right,
     double bottom,
     String label,
+    String hint,
+    String value,
     int textDirection,
     Float64List transform,
     Int32List children

engine/src/flutter/lib/ui/semantics/semantics_node.h

@@ -52,6 +52,8 @@ struct SemanticsNode {
   int32_t flags = 0;
   int32_t actions = 0;
   std::string label;
+  std::string hint;
+  std::string value;
   int32_t textDirection = 0;  // 0=unknown, 1=rtl, 2=ltr
 
   SkRect rect = SkRect::MakeEmpty();

engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc

@@ -42,6 +42,8 @@ void SemanticsUpdateBuilder::updateNode(int id,
                                         double right,
                                         double bottom,
                                         std::string label,
+                                        std::string hint,
+                                        std::string value,
                                         int textDirection,
                                         const tonic::Float64List& transform,
                                         const tonic::Int32List& children) {
@@ -51,6 +53,8 @@ void SemanticsUpdateBuilder::updateNode(int id,
   node.actions = actions;
   node.rect = SkRect::MakeLTRB(left, top, right, bottom);
   node.label = label;
+  node.hint = hint;
+  node.value = value;
   node.textDirection = textDirection;
   node.transform.setColMajord(transform.data());
   node.children = std::vector<int32_t>(

engine/src/flutter/lib/ui/semantics/semantics_update_builder.h

@@ -33,6 +33,8 @@ class SemanticsUpdateBuilder
                   double right,
                   double bottom,
                   std::string label,
+                  std::string hint,
+                  std::string value,
                   int textDirection,
                   const tonic::Float64List& transform,
                   const tonic::Int32List& children);

engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java

@@ -158,7 +158,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         result.setCheckable((object.flags & SEMANTICS_FLAG_HAS_CHECKED_STATE) != 0);
         result.setChecked((object.flags & SEMANTICS_FLAG_IS_CHECKED) != 0);
         result.setSelected((object.flags & SEMANTICS_FLAG_IS_SELECTED) != 0);
-        result.setText(object.label);
+        result.setText(object.getValueLabelHint());
 
         if ((object.flags & SEMANTICS_FLAG_IS_BUTTON) != 0) {
           result.setClassName("android.widget.Button");
@@ -434,6 +434,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         int flags;
         int actions;
         String label;
+        String value;
+        String hint;
         TextDirection textDirection;
 
         private float left;
@@ -468,11 +470,14 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
             flags = buffer.getInt();
             actions = buffer.getInt();
 
-            final int stringIndex = buffer.getInt();
-            if (stringIndex == -1)
-                label = null;
-            else
-                label = strings[stringIndex];
+            int stringIndex = buffer.getInt();
+            label = stringIndex == -1 ? null : strings[stringIndex];
+
+            stringIndex = buffer.getInt();
+            value = stringIndex == -1 ? null : strings[stringIndex];
+
+            stringIndex = buffer.getInt();
+            hint = stringIndex == -1 ? null : strings[stringIndex];
 
             textDirection = TextDirection.fromInt(buffer.getInt());
 
@@ -620,5 +625,18 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         private float max(float a, float b, float c, float d) {
             return Math.max(a, Math.max(b, Math.max(c, d)));
         }
+
+        private String getValueLabelHint() {
+            StringBuilder sb = new StringBuilder();
+            String[] array = { value, label, hint };
+            for (String word: array) {
+                if (word != null && (word = word.trim()).length() > 0) {
+                    if (sb.length() > 0)
+                        sb.append(", ");
+                    sb.append(word);
+                }
+            }
+            return sb.length() > 0 ? sb.toString() : null;
+        }
     }
 }

engine/src/flutter/shell/platform/android/platform_view_android.cc

@@ -431,7 +431,7 @@ bool PlatformViewAndroid::ResourceContextMakeCurrent() {
 
 void PlatformViewAndroid::UpdateSemantics(
     std::vector<blink::SemanticsNode> update) {
-  constexpr size_t kBytesPerNode = 26 * sizeof(int32_t);
+  constexpr size_t kBytesPerNode = 28 * sizeof(int32_t);
   constexpr size_t kBytesPerChild = sizeof(int32_t);
 
   JNIEnv* env = fml::jni::AttachCurrentThread();
@@ -464,6 +464,18 @@ void PlatformViewAndroid::UpdateSemantics(
         buffer_int32[position++] = strings.size();
         strings.push_back(node.label);
       }
+      if (node.value.empty()) {
+        buffer_int32[position++] = -1;
+      } else {
+        buffer_int32[position++] = strings.size();
+        strings.push_back(node.value);
+      }
+      if (node.hint.empty()) {
+        buffer_int32[position++] = -1;
+      } else {
+        buffer_int32[position++] = strings.size();
+        strings.push_back(node.hint);
+      }
       buffer_int32[position++] = node.textDirection;
       buffer_float32[position++] = node.rect.left();
       buffer_float32[position++] = node.rect.top();

engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm

@@ -162,6 +162,18 @@ bool GeometryComparator(SemanticsObject* a, SemanticsObject* b) {
   return @(_node.label.data());
 }
 
+- (NSString*)accessibilityHint {
+  if (_node.hint.empty())
+    return nil;
+  return @(_node.hint.data());
+}
+
+- (NSString*)accessibilityValue {
+  if (_node.value.empty())
+    return nil;
+  return @(_node.value.data());
+}
+
 - (UIAccessibilityTraits)accessibilityTraits {
   UIAccessibilityTraits traits = UIAccessibilityTraitNone;
   if (_node.HasAction(blink::SemanticsAction::kIncrease) ||