From 394c11e33fc08f52d8de091fbce4e0ba8eb710b8 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 30 Jan 2018 13:42:12 -0800 Subject: [PATCH] Adding semantics traversal order support (flutter/engine#4540) This adds support for semantics traversal ordering. It is a companion to flutter/flutter#14060, adding support for a sortIndex in the semantics data passed to the engine. Addresses flutter/flutter#12187 --- engine/src/flutter/lib/ui/semantics.dart | 41 ++++++++++--------- .../flutter/lib/ui/semantics/semantics_node.h | 1 + .../ui/semantics/semantics_update_builder.cc | 2 + .../ui/semantics/semantics_update_builder.h | 1 + engine/src/flutter/lib/ui/window.dart | 2 +- .../io/flutter/view/AccessibilityBridge.java | 24 +++++------ .../platform/android/platform_view_android.cc | 6 ++- 7 files changed, 41 insertions(+), 36 deletions(-) diff --git a/engine/src/flutter/lib/ui/semantics.dart b/engine/src/flutter/lib/ui/semantics.dart index 0bd09a1e19..71f60b9959 100644 --- a/engine/src/flutter/lib/ui/semantics.dart +++ b/engine/src/flutter/lib/ui/semantics.dart @@ -340,7 +340,7 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 { /// The `rect` is the region occupied by this node in its own coordinate /// system. /// - /// The `transform` is a matrix that maps this node's coodinate system into + /// The `transform` is a matrix that maps this node's coordinate system into /// its parent's coordinate system. void updateNode({ int id, @@ -355,30 +355,30 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 { String increasedValue, String decreasedValue, TextDirection textDirection, + int nextNodeId, Float64List transform, Int32List children, }) { if (transform.length != 16) throw new ArgumentError('transform argument must have 16 entries.'); - _updateNode( - id, - flags, - actions, - textSelectionBase, - textSelectionExtent, - rect.left, - rect.top, - rect.right, - rect.bottom, - label, - hint, - value, - increasedValue, - decreasedValue, - textDirection != null ? textDirection.index + 1 : 0, - transform, - children, - ); + _updateNode(id, + flags, + actions, + textSelectionBase, + textSelectionExtent, + rect.left, + rect.top, + rect.right, + rect.bottom, + label, + hint, + value, + increasedValue, + decreasedValue, + textDirection != null ? textDirection.index + 1 : 0, + nextNodeId ?? -1, + transform, + children,); } void _updateNode( int id, @@ -396,6 +396,7 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 { String increasedValue, String decreasedValue, int textDirection, + int nextNodeId, Float64List transform, Int32List children, ) native 'SemanticsUpdateBuilder_updateNode'; diff --git a/engine/src/flutter/lib/ui/semantics/semantics_node.h b/engine/src/flutter/lib/ui/semantics/semantics_node.h index c1cc027bb7..51ad3d832b 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_node.h +++ b/engine/src/flutter/lib/ui/semantics/semantics_node.h @@ -63,6 +63,7 @@ struct SemanticsNode { std::string increasedValue; std::string decreasedValue; int32_t textDirection = 0; // 0=unknown, 1=rtl, 2=ltr + int32_t nextNodeId = -1; SkRect rect = SkRect::MakeEmpty(); SkMatrix44 transform = SkMatrix44(SkMatrix44::kIdentity_Constructor); diff --git a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc index 46d722552c..81ba0671d7 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc +++ b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.cc @@ -49,6 +49,7 @@ void SemanticsUpdateBuilder::updateNode(int id, std::string increasedValue, std::string decreasedValue, int textDirection, + int nextNodeId, const tonic::Float64List& transform, const tonic::Int32List& children) { SemanticsNode node; @@ -64,6 +65,7 @@ void SemanticsUpdateBuilder::updateNode(int id, node.increasedValue = increasedValue; node.decreasedValue = decreasedValue; node.textDirection = textDirection; + node.nextNodeId = nextNodeId; node.transform.setColMajord(transform.data()); node.children = std::vector( children.data(), children.data() + children.num_elements()); diff --git a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h index 7c85b35abc..c27ca702fc 100644 --- a/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h +++ b/engine/src/flutter/lib/ui/semantics/semantics_update_builder.h @@ -40,6 +40,7 @@ class SemanticsUpdateBuilder std::string increasedValue, std::string decreasedValue, int textDirection, + int nextNodeId, const tonic::Float64List& transform, const tonic::Int32List& children); diff --git a/engine/src/flutter/lib/ui/window.dart b/engine/src/flutter/lib/ui/window.dart index e3ae38fd82..03f9009ab5 100644 --- a/engine/src/flutter/lib/ui/window.dart +++ b/engine/src/flutter/lib/ui/window.dart @@ -342,7 +342,7 @@ class Window { /// /// When this changes, [onMetricsChanged] is called. /// - /// At startup, the size of the application window may not be know before Dart + /// At startup, the size of the application window may not be known before Dart /// code runs. If this value is observed early in the application lifecycle, /// it may report [Size.zero]. /// diff --git a/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java index f068d8fe72..04aa701e93 100644 --- a/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/engine/src/flutter/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -236,6 +236,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess result.setSelected(object.hasFlag(Flag.IS_SELECTED)); result.setText(object.getValueLabelHint()); + result.setTraversalBefore(mOwner, + object.nextNodeId == -1 ? View.NO_ID : object.nextNodeId); // Accessibility Focus if (mA11yFocusedObject != null && mA11yFocusedObject.id == virtualViewId) { @@ -245,16 +247,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess } if (object.children != null) { - List childrenInTraversalOrder = - new ArrayList(object.children); - Collections.sort(childrenInTraversalOrder, new Comparator() { - public int compare(SemanticsObject a, SemanticsObject b) { - final int top = Integer.compare(a.globalRect.top, b.globalRect.top); - // TODO(goderbauer): sort right-to-left in rtl environments. - return top == 0 ? Integer.compare(a.globalRect.left, b.globalRect.left) : top; - } - }); - for (SemanticsObject child : childrenInTraversalOrder) { + for (SemanticsObject child : object.children) { result.addChild(mOwner, child.id); } } @@ -673,6 +666,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess String decreasedValue; String hint; TextDirection textDirection; + int nextNodeId; boolean hadPreviousConfig = false; int previousFlags; @@ -709,14 +703,16 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess return (previousFlags & flag.value) != 0; } - void log(String indent) { + void log(String indent, boolean recursive) { Log.i(TAG, indent + "SemanticsObject id=" + id + " label=" + label + " actions=" + actions + " flags=" + flags + "\n" + + indent + " +-- textDirection=" + textDirection + "\n"+ + indent + " +-- nextNodeId=" + nextNodeId + "\n"+ indent + " +-- rect.ltrb=(" + left + ", " + top + ", " + right + ", " + bottom + ")\n" + indent + " +-- transform=" + Arrays.toString(transform) + "\n"); - if (children != null) { + if (children != null && recursive) { String childIndent = indent + " "; for (SemanticsObject child : children) { - child.log(childIndent); + child.log(childIndent, recursive); } } } @@ -750,6 +746,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess textDirection = TextDirection.fromInt(buffer.getInt()); + nextNodeId = buffer.getInt(); + left = buffer.getFloat(); top = buffer.getFloat(); right = buffer.getFloat(); diff --git a/engine/src/flutter/shell/platform/android/platform_view_android.cc b/engine/src/flutter/shell/platform/android/platform_view_android.cc index fd58baed5a..65833e34ec 100644 --- a/engine/src/flutter/shell/platform/android/platform_view_android.cc +++ b/engine/src/flutter/shell/platform/android/platform_view_android.cc @@ -461,7 +461,7 @@ bool PlatformViewAndroid::ResourceContextMakeCurrent() { void PlatformViewAndroid::UpdateSemantics( std::vector update) { - constexpr size_t kBytesPerNode = 32 * sizeof(int32_t); + constexpr size_t kBytesPerNode = 33 * sizeof(int32_t); constexpr size_t kBytesPerChild = sizeof(int32_t); JNIEnv* env = fml::jni::AttachCurrentThread(); @@ -484,7 +484,8 @@ void PlatformViewAndroid::UpdateSemantics( size_t position = 0; for (const blink::SemanticsNode& node : update) { // If you edit this code, make sure you update kBytesPerNode - // above to match the number of values you are sending. + // and/or kBytesPerChild above to match the number of values you are + // sending. buffer_int32[position++] = node.id; buffer_int32[position++] = node.flags; buffer_int32[position++] = node.actions; @@ -521,6 +522,7 @@ void PlatformViewAndroid::UpdateSemantics( strings.push_back(node.hint); } buffer_int32[position++] = node.textDirection; + buffer_int32[position++] = node.nextNodeId; buffer_float32[position++] = node.rect.left(); buffer_float32[position++] = node.rect.top(); buffer_float32[position++] = node.rect.right();