From 88b2b4b5aa26dfd5d14fcd14188b9a17c9e1b8d0 Mon Sep 17 00:00:00 2001 From: Michael Goderbauer Date: Mon, 23 Oct 2017 16:46:01 -0700 Subject: [PATCH] Support for accessibility label and hint (flutter/engine#4264) * Support for accessibility label and hint * review comments --- engine/src/flutter/lib/ui/semantics.dart | 12 ++++++-- .../flutter/lib/ui/semantics/semantics_node.h | 2 ++ .../ui/semantics/semantics_update_builder.cc | 4 +++ .../ui/semantics/semantics_update_builder.h | 2 ++ .../io/flutter/view/AccessibilityBridge.java | 30 +++++++++++++++---- .../platform/android/platform_view_android.cc | 14 ++++++++- .../framework/Source/accessibility_bridge.mm | 12 ++++++++ 7 files changed, 67 insertions(+), 9 deletions(-) diff --git a/engine/src/flutter/lib/ui/semantics.dart b/engine/src/flutter/lib/ui/semantics.dart index 09d92cd0b2..3584442880 100644 --- a/engine/src/flutter/lib/ui/semantics.dart +++ b/engine/src/flutter/lib/ui/semantics.dart @@ -214,8 +214,10 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 { /// asynchronously, the [Window.onSemanticsAction] callback might be called /// with an action that is no longer possible. /// - /// The `label` is a string that describes this node. Its reading direction is - /// given by `textDirection`. + /// The `label` is a string that describes this node. The `value` property + /// describes the current value of the node as a string. The `hint` string + /// describes what result an action performed on this node has. The reading + /// direction of all these strings is given by `textDirection`. /// /// The `rect` is the region occupied by this node in its own coordinate /// system. @@ -228,6 +230,8 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 { int actions, Rect rect, String label, + String hint, + String value, TextDirection textDirection, Float64List transform, Int32List children @@ -242,6 +246,8 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 { rect.right, rect.bottom, label, + hint, + value, textDirection != null ? textDirection.index + 1 : 0, transform, children); @@ -255,6 +261,8 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 { double right, double bottom, String label, + String hint, + String value, int textDirection, Float64List transform, Int32List children diff --git a/engine/src/flutter/lib/ui/semantics/semantics_node.h b/engine/src/flutter/lib/ui/semantics/semantics_node.h index 347152dc8b..3c70ac3951 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_node.h +++ b/engine/src/flutter/lib/ui/semantics/semantics_node.h @@ -52,6 +52,8 @@ struct SemanticsNode { int32_t flags = 0; int32_t actions = 0; std::string label; + std::string hint; + std::string value; int32_t textDirection = 0; // 0=unknown, 1=rtl, 2=ltr SkRect rect = SkRect::MakeEmpty(); diff --git a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc index 265a91c805..83d6121db0 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc +++ b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc @@ -42,6 +42,8 @@ void SemanticsUpdateBuilder::updateNode(int id, double right, double bottom, std::string label, + std::string hint, + std::string value, int textDirection, const tonic::Float64List& transform, const tonic::Int32List& children) { @@ -51,6 +53,8 @@ void SemanticsUpdateBuilder::updateNode(int id, node.actions = actions; node.rect = SkRect::MakeLTRB(left, top, right, bottom); node.label = label; + node.hint = hint; + node.value = value; node.textDirection = textDirection; node.transform.setColMajord(transform.data()); node.children = std::vector( diff --git a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h index c63e045de5..23cabfc63e 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h +++ b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h @@ -33,6 +33,8 @@ class SemanticsUpdateBuilder double right, double bottom, std::string label, + std::string hint, + std::string value, int textDirection, const tonic::Float64List& transform, const tonic::Int32List& children); diff --git a/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java index afeb0c57ef..7dee39990a 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -158,7 +158,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess result.setCheckable((object.flags & SEMANTICS_FLAG_HAS_CHECKED_STATE) != 0); result.setChecked((object.flags & SEMANTICS_FLAG_IS_CHECKED) != 0); result.setSelected((object.flags & SEMANTICS_FLAG_IS_SELECTED) != 0); - result.setText(object.label); + result.setText(object.getValueLabelHint()); if ((object.flags & SEMANTICS_FLAG_IS_BUTTON) != 0) { result.setClassName("android.widget.Button"); @@ -434,6 +434,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess int flags; int actions; String label; + String value; + String hint; TextDirection textDirection; private float left; @@ -468,11 +470,14 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess flags = buffer.getInt(); actions = buffer.getInt(); - final int stringIndex = buffer.getInt(); - if (stringIndex == -1) - label = null; - else - label = strings[stringIndex]; + int stringIndex = buffer.getInt(); + label = stringIndex == -1 ? null : strings[stringIndex]; + + stringIndex = buffer.getInt(); + value = stringIndex == -1 ? null : strings[stringIndex]; + + stringIndex = buffer.getInt(); + hint = stringIndex == -1 ? null : strings[stringIndex]; textDirection = TextDirection.fromInt(buffer.getInt()); @@ -620,5 +625,18 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess private float max(float a, float b, float c, float d) { return Math.max(a, Math.max(b, Math.max(c, d))); } + + private String getValueLabelHint() { + StringBuilder sb = new StringBuilder(); + String[] array = { value, label, hint }; + for (String word: array) { + if (word != null && (word = word.trim()).length() > 0) { + if (sb.length() > 0) + sb.append(", "); + sb.append(word); + } + } + return sb.length() > 0 ? sb.toString() : null; + } } } diff --git a/engine/src/flutter/shell/platform/android/platform_view_android.cc b/engine/src/flutter/shell/platform/android/platform_view_android.cc index 55a8f0306c..d4412a6130 100644 --- a/engine/src/flutter/shell/platform/android/platform_view_android.cc +++ b/engine/src/flutter/shell/platform/android/platform_view_android.cc @@ -431,7 +431,7 @@ bool PlatformViewAndroid::ResourceContextMakeCurrent() { void PlatformViewAndroid::UpdateSemantics( std::vector update) { - constexpr size_t kBytesPerNode = 26 * sizeof(int32_t); + constexpr size_t kBytesPerNode = 28 * sizeof(int32_t); constexpr size_t kBytesPerChild = sizeof(int32_t); JNIEnv* env = fml::jni::AttachCurrentThread(); @@ -464,6 +464,18 @@ void PlatformViewAndroid::UpdateSemantics( buffer_int32[position++] = strings.size(); strings.push_back(node.label); } + if (node.value.empty()) { + buffer_int32[position++] = -1; + } else { + buffer_int32[position++] = strings.size(); + strings.push_back(node.value); + } + if (node.hint.empty()) { + buffer_int32[position++] = -1; + } else { + buffer_int32[position++] = strings.size(); + strings.push_back(node.hint); + } buffer_int32[position++] = node.textDirection; buffer_float32[position++] = node.rect.left(); buffer_float32[position++] = node.rect.top(); diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm index 7167ce049b..a655ef85c5 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm @@ -162,6 +162,18 @@ bool GeometryComparator(SemanticsObject* a, SemanticsObject* b) { return @(_node.label.data()); } +- (NSString*)accessibilityHint { + if (_node.hint.empty()) + return nil; + return @(_node.hint.data()); +} + +- (NSString*)accessibilityValue { + if (_node.value.empty()) + return nil; + return @(_node.value.data()); +} + - (UIAccessibilityTraits)accessibilityTraits { UIAccessibilityTraits traits = UIAccessibilityTraitNone; if (_node.HasAction(blink::SemanticsAction::kIncrease) ||