Forward a11y events from Hybrid Composition overlays (flutter/engine#36924)

This commit is contained in:
stuartmorgan
2022-10-31 06:54:14 -07:00
committed by GitHub
parent 03585a78cc
commit 6013743aae
11 changed files with 328 additions and 22 deletions

View File

@@ -2277,6 +2277,7 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/editing/TextInpu
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/localization/LocalizationPlugin.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/mouse/MouseCursorPlugin.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/AccessibilityEventsDelegate.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformOverlayView.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformPlugin.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformView.java
FILE: ../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewFactory.java

View File

@@ -279,6 +279,7 @@ android_java_sources = [
"io/flutter/plugin/localization/LocalizationPlugin.java",
"io/flutter/plugin/mouse/MouseCursorPlugin.java",
"io/flutter/plugin/platform/AccessibilityEventsDelegate.java",
"io/flutter/plugin/platform/PlatformOverlayView.java",
"io/flutter/plugin/platform/PlatformPlugin.java",
"io/flutter/plugin/platform/PlatformView.java",
"io/flutter/plugin/platform/PlatformViewFactory.java",

View File

@@ -4,6 +4,7 @@
package io.flutter.plugin.platform;
import android.view.MotionEvent;
import android.view.View;
import android.view.accessibility.AccessibilityEvent;
import androidx.annotation.NonNull;
@@ -43,6 +44,13 @@ class AccessibilityEventsDelegate {
embeddedView, eventOrigin, event);
}
public boolean onAccessibilityHoverEvent(MotionEvent event, boolean ignorePlatformViews) {
if (accessibilityBridge == null) {
return false;
}
return accessibilityBridge.onAccessibilityHoverEvent(event, ignorePlatformViews);
}
/*
* This setter should only be used directly in PlatformViewsController when attached/detached to an accessibility
* bridge.

View File

@@ -0,0 +1,49 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package io.flutter.plugin.platform;
import android.content.Context;
import android.util.AttributeSet;
import android.view.MotionEvent;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import io.flutter.embedding.android.FlutterImageView;
/** A host view for Flutter content displayed over a platform view. */
public class PlatformOverlayView extends FlutterImageView {
@Nullable private AccessibilityEventsDelegate accessibilityDelegate;
public PlatformOverlayView(
@NonNull Context context,
int width,
int height,
@NonNull AccessibilityEventsDelegate accessibilityDelegate) {
super(context, width, height, FlutterImageView.SurfaceKind.overlay);
this.accessibilityDelegate = accessibilityDelegate;
}
public PlatformOverlayView(@NonNull Context context) {
this(context, 1, 1, null);
}
public PlatformOverlayView(@NonNull Context context, @NonNull AttributeSet attrs) {
this(context, 1, 1, null);
}
@Override
public boolean onHoverEvent(@NonNull MotionEvent event) {
// This view doesn't have any accessibility information of its own, but anything drawn in
// this view is visible above the platform view it is overlaying, so should respond to
// accessibility exploration events. Forward those events to the accessibility delegate in
// a special mode that will stop as soon as it reaches a platform view, so that it will not
// find widgets that behind the platform view. If no such widget is found, treat the event
// as unhandled so that it can fall through to the platform view.
if (accessibilityDelegate != null
&& accessibilityDelegate.onAccessibilityHoverEvent(event, true)) {
return true;
}
return super.onHoverEvent(event);
}
}

View File

@@ -23,7 +23,6 @@ import androidx.annotation.UiThread;
import androidx.annotation.VisibleForTesting;
import io.flutter.Log;
import io.flutter.embedding.android.AndroidTouchProcessor;
import io.flutter.embedding.android.FlutterImageView;
import io.flutter.embedding.android.FlutterView;
import io.flutter.embedding.android.MotionEventTracker;
import io.flutter.embedding.engine.FlutterOverlaySurface;
@@ -115,7 +114,7 @@ public class PlatformViewsController implements PlatformViewsAccessibilityDelega
private final SparseArray<FlutterMutatorView> platformViewParent;
// Map of unique IDs to views that render overlay layers.
private final SparseArray<FlutterImageView> overlayLayerViews;
private final SparseArray<PlatformOverlayView> overlayLayerViews;
// The platform view wrappers that are appended to FlutterView.
//
@@ -1136,7 +1135,7 @@ public class PlatformViewsController implements PlatformViewsAccessibilityDelega
}
initializeRootImageViewIfNeeded();
final FlutterImageView overlayView = overlayLayerViews.get(id);
final PlatformOverlayView overlayView = overlayLayerViews.get(id);
if (overlayView.getParent() == null) {
flutterView.addView(overlayView);
}
@@ -1191,7 +1190,7 @@ public class PlatformViewsController implements PlatformViewsAccessibilityDelega
private void finishFrame(boolean isFrameRenderedUsingImageReaders) {
for (int i = 0; i < overlayLayerViews.size(); i++) {
final int overlayId = overlayLayerViews.keyAt(i);
final FlutterImageView overlayView = overlayLayerViews.valueAt(i);
final PlatformOverlayView overlayView = overlayLayerViews.valueAt(i);
if (currentFrameUsedOverlayLayerIds.contains(overlayId)) {
flutterView.attachOverlaySurfaceToRender(overlayView);
@@ -1241,14 +1240,14 @@ public class PlatformViewsController implements PlatformViewsAccessibilityDelega
@VisibleForTesting
@TargetApi(19)
@NonNull
public FlutterOverlaySurface createOverlaySurface(@NonNull FlutterImageView imageView) {
public FlutterOverlaySurface createOverlaySurface(@NonNull PlatformOverlayView imageView) {
final int id = nextOverlayLayerId++;
overlayLayerViews.put(id, imageView);
return new FlutterOverlaySurface(id, imageView.getSurface());
}
/**
* Creates an overlay surface while the Flutter view is rendered by {@code FlutterImageView}.
* Creates an overlay surface while the Flutter view is rendered by {@code PlatformOverlayView}.
*
* <p>This method is invoked by {@code FlutterJNI} only.
*
@@ -1264,11 +1263,11 @@ public class PlatformViewsController implements PlatformViewsAccessibilityDelega
//
// The final view size is determined when its frame is set.
return createOverlaySurface(
new FlutterImageView(
new PlatformOverlayView(
flutterView.getContext(),
flutterView.getWidth(),
flutterView.getHeight(),
FlutterImageView.SurfaceKind.overlay));
accessibilityEventsDelegate));
}
/**
@@ -1278,7 +1277,7 @@ public class PlatformViewsController implements PlatformViewsAccessibilityDelega
*/
public void destroyOverlaySurfaces() {
for (int viewId = 0; viewId < overlayLayerViews.size(); viewId++) {
final FlutterImageView overlayView = overlayLayerViews.valueAt(viewId);
final PlatformOverlayView overlayView = overlayLayerViews.valueAt(viewId);
overlayView.detachFromRenderer();
overlayView.closeImageReader();
// Don't remove overlayView from the view hierarchy since this method can

View File

@@ -1488,6 +1488,24 @@ public class AccessibilityBridge extends AccessibilityNodeProvider {
* View#onHoverEvent(MotionEvent)}.
*/
public boolean onAccessibilityHoverEvent(MotionEvent event) {
return onAccessibilityHoverEvent(event, false);
}
/**
* A hover {@link MotionEvent} has occurred in the {@code View} that corresponds to this {@code
* AccessibilityBridge}.
*
* <p>If {@code ignorePlatformViews} is true, if hit testing for the event finds a platform view,
* the event will not be handled. This is useful when handling accessibility events for views
* overlaying platform views. See {@code PlatformOverlayView} for details.
*
* <p>This method returns true if Flutter's accessibility system handled the hover event, false
* otherwise.
*
* <p>This method should be invoked from the corresponding {@code View}'s {@link
* View#onHoverEvent(MotionEvent)}.
*/
public boolean onAccessibilityHoverEvent(MotionEvent event, boolean ignorePlatformViews) {
if (!accessibilityManager.isTouchExplorationEnabled()) {
return false;
}
@@ -1496,17 +1514,21 @@ public class AccessibilityBridge extends AccessibilityNodeProvider {
}
SemanticsNode semanticsNodeUnderCursor =
getRootSemanticsNode().hitTest(new float[] {event.getX(), event.getY(), 0, 1});
getRootSemanticsNode()
.hitTest(new float[] {event.getX(), event.getY(), 0, 1}, ignorePlatformViews);
// semanticsNodeUnderCursor can be null when hovering over non-flutter UI such as
// the Android navigation bar due to hitTest() bounds checking.
if (semanticsNodeUnderCursor != null && semanticsNodeUnderCursor.platformViewId != -1) {
if (ignorePlatformViews) {
return false;
}
return accessibilityViewEmbedder.onAccessibilityHoverEvent(
semanticsNodeUnderCursor.id, event);
}
if (event.getAction() == MotionEvent.ACTION_HOVER_ENTER
|| event.getAction() == MotionEvent.ACTION_HOVER_MOVE) {
handleTouchExploration(event.getX(), event.getY());
handleTouchExploration(event.getX(), event.getY(), ignorePlatformViews);
} else if (event.getAction() == MotionEvent.ACTION_HOVER_EXIT) {
onTouchExplorationExit();
} else {
@@ -1539,12 +1561,12 @@ public class AccessibilityBridge extends AccessibilityNodeProvider {
* a {@link AccessibilityEvent#TYPE_VIEW_HOVER_ENTER} event for the new hover node, followed by a
* {@link AccessibilityEvent#TYPE_VIEW_HOVER_EXIT} event for the old hover node.
*/
private void handleTouchExploration(float x, float y) {
private void handleTouchExploration(float x, float y, boolean ignorePlatformViews) {
if (flutterSemanticsTree.isEmpty()) {
return;
}
SemanticsNode semanticsNodeUnderCursor =
getRootSemanticsNode().hitTest(new float[] {x, y, 0, 1});
getRootSemanticsNode().hitTest(new float[] {x, y, 0, 1}, ignorePlatformViews);
if (semanticsNodeUnderCursor != hoveredObject) {
// sending ENTER before EXIT is how Android wants it
if (semanticsNodeUnderCursor != null) {
@@ -2619,7 +2641,15 @@ public class AccessibilityBridge extends AccessibilityNodeProvider {
return globalRect;
}
private SemanticsNode hitTest(float[] point) {
/**
* Hit tests {@code point} to find the deepest focusable node in the node tree at that point.
*
* @param point The point to hit test against this node.
* @param stopAtPlatformView Whether to return a platform view if found, regardless of whether
* or not it is focusable.
* @return The found node, or null if no relevant node was found at the given point.
*/
private SemanticsNode hitTest(float[] point, boolean stopAtPlatformView) {
final float w = point[3];
final float x = point[0] / w;
final float y = point[1] / w;
@@ -2631,12 +2661,13 @@ public class AccessibilityBridge extends AccessibilityNodeProvider {
}
child.ensureInverseTransform();
Matrix.multiplyMV(transformedPoint, 0, child.inverseTransform, 0, point, 0);
final SemanticsNode result = child.hitTest(transformedPoint);
final SemanticsNode result = child.hitTest(transformedPoint, stopAtPlatformView);
if (result != null) {
return result;
}
}
return isFocusable() ? this : null;
final boolean foundPlatformView = stopAtPlatformView && platformViewId != -1;
return isFocusable() || foundPlatformView ? this : null;
}
// TODO(goderbauer): This should be decided by the framework once we have more information

View File

@@ -0,0 +1,83 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package io.flutter.plugin.platform;
import static junit.framework.TestCase.assertFalse;
import static junit.framework.TestCase.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import android.view.MotionEvent;
import android.view.View;
import android.view.accessibility.AccessibilityEvent;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import io.flutter.view.AccessibilityBridge;
import org.junit.Test;
import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class)
public class AccessibilityEventsDelegateTest {
@Test
public void acessibilityEventsDelegate_forwardsAccessibilityEvents() {
final AccessibilityBridge mockAccessibilityBridge = mock(AccessibilityBridge.class);
final View embeddedView = mock(View.class);
final View originView = mock(View.class);
final AccessibilityEvent event = mock(AccessibilityEvent.class);
AccessibilityEventsDelegate delegate = new AccessibilityEventsDelegate();
delegate.setAccessibilityBridge(mockAccessibilityBridge);
when(mockAccessibilityBridge.externalViewRequestSendAccessibilityEvent(any(), any(), any()))
.thenReturn(true);
final boolean handled = delegate.requestSendAccessibilityEvent(embeddedView, originView, event);
assertTrue(handled);
verify(mockAccessibilityBridge, times(1))
.externalViewRequestSendAccessibilityEvent(embeddedView, originView, event);
}
@Test
public void acessibilityEventsDelegate_withoutBridge_noopsAccessibilityEvents() {
final View embeddedView = mock(View.class);
final View originView = mock(View.class);
final AccessibilityEvent event = mock(AccessibilityEvent.class);
AccessibilityEventsDelegate delegate = new AccessibilityEventsDelegate();
final boolean handled = delegate.requestSendAccessibilityEvent(embeddedView, originView, event);
assertFalse(handled);
}
@Test
public void acessibilityEventsDelegate_forwardsHoverEvents() {
final AccessibilityBridge mockAccessibilityBridge = mock(AccessibilityBridge.class);
final MotionEvent event = mock(MotionEvent.class);
AccessibilityEventsDelegate delegate = new AccessibilityEventsDelegate();
delegate.setAccessibilityBridge(mockAccessibilityBridge);
when(mockAccessibilityBridge.onAccessibilityHoverEvent(any(), anyBoolean())).thenReturn(true);
final boolean handled = delegate.onAccessibilityHoverEvent(event, true);
assertTrue(handled);
verify(mockAccessibilityBridge, times(1)).onAccessibilityHoverEvent(event, true);
}
@Test
public void acessibilityEventsDelegate_withoutBridge_noopsHoverEvents() {
final MotionEvent event = mock(MotionEvent.class);
AccessibilityEventsDelegate delegate = new AccessibilityEventsDelegate();
final boolean handled = delegate.onAccessibilityHoverEvent(event, true);
assertFalse(handled);
}
}

View File

@@ -0,0 +1,47 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package io.flutter.plugin.platform;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import android.content.Context;
import android.os.SystemClock;
import android.view.MotionEvent;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Test;
import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class)
public class PlatformOverlayViewTest {
private final Context ctx = ApplicationProvider.getApplicationContext();
@Test
public void platformOverlayView_forwardsHover() {
final AccessibilityEventsDelegate mockAccessibilityDelegate =
mock(AccessibilityEventsDelegate.class);
when(mockAccessibilityDelegate.onAccessibilityHoverEvent(any(), eq(true))).thenReturn(true);
final int size = 10;
final PlatformOverlayView imageView =
new PlatformOverlayView(ctx, size, size, mockAccessibilityDelegate);
MotionEvent event =
MotionEvent.obtain(
SystemClock.uptimeMillis(),
SystemClock.uptimeMillis(),
MotionEvent.ACTION_HOVER_MOVE,
size / 2,
size / 2,
0);
imageView.onHoverEvent(event);
verify(mockAccessibilityDelegate, times(1)).onAccessibilityHoverEvent(event, true);
}
}

View File

@@ -812,7 +812,7 @@ public class PlatformViewsControllerTest {
/* viewHeight=*/ 10,
/* mutatorsStack=*/ new FlutterMutatorsStack());
final FlutterImageView overlayImageView = mock(FlutterImageView.class);
final PlatformOverlayView overlayImageView = mock(PlatformOverlayView.class);
when(overlayImageView.acquireLatestImage()).thenReturn(true);
final FlutterOverlaySurface overlaySurface =
@@ -955,7 +955,7 @@ public class PlatformViewsControllerTest {
/* viewHeight=*/ 10,
/* mutatorsStack=*/ new FlutterMutatorsStack());
final FlutterImageView overlayImageView = mock(FlutterImageView.class);
final PlatformOverlayView overlayImageView = mock(PlatformOverlayView.class);
when(overlayImageView.acquireLatestImage()).thenReturn(true);
final FlutterOverlaySurface overlaySurface =
@@ -992,7 +992,7 @@ public class PlatformViewsControllerTest {
final FlutterView flutterView = mock(FlutterView.class);
platformViewsController.attachToView(flutterView);
final FlutterImageView overlayImageView = mock(FlutterImageView.class);
final PlatformOverlayView overlayImageView = mock(PlatformOverlayView.class);
when(overlayImageView.acquireLatestImage()).thenReturn(true);
final FlutterOverlaySurface overlaySurface =
@@ -1030,7 +1030,7 @@ public class PlatformViewsControllerTest {
final FlutterView flutterView = mock(FlutterView.class);
platformViewsController.attachToView(flutterView);
final FlutterImageView overlayImageView = mock(FlutterImageView.class);
final PlatformOverlayView overlayImageView = mock(PlatformOverlayView.class);
when(overlayImageView.acquireLatestImage()).thenReturn(true);
final FlutterOverlaySurface overlaySurface =
@@ -1068,7 +1068,7 @@ public class PlatformViewsControllerTest {
final FlutterView flutterView = mock(FlutterView.class);
platformViewsController.attachToView(flutterView);
final FlutterImageView overlayImageView = mock(FlutterImageView.class);
final PlatformOverlayView overlayImageView = mock(PlatformOverlayView.class);
when(overlayImageView.acquireLatestImage()).thenReturn(true);
final FlutterOverlaySurface overlaySurface =
@@ -1175,7 +1175,7 @@ public class PlatformViewsControllerTest {
/* mutatorsStack=*/ new FlutterMutatorsStack());
assertEquals(flutterView.getChildCount(), 2);
assertTrue(!(flutterView.getChildAt(0) instanceof FlutterImageView));
assertTrue(!(flutterView.getChildAt(0) instanceof PlatformOverlayView));
assertTrue(flutterView.getChildAt(1) instanceof FlutterMutatorView);
// Simulate dispose call from the framework.

View File

@@ -571,6 +571,92 @@ public class AccessibilityBridgeTest {
assertEquals(accessibilityBridge.getHoveredObjectId(), 2);
}
@Test
public void itFindsPlatformViewsDuringHoverByDefault() {
AccessibilityViewEmbedder mockViewEmbedder = mock(AccessibilityViewEmbedder.class);
AccessibilityManager mockManager = mock(AccessibilityManager.class);
View mockRootView = mock(View.class);
Context context = mock(Context.class);
when(mockRootView.getContext()).thenReturn(context);
when(context.getPackageName()).thenReturn("test");
AccessibilityBridge accessibilityBridge =
setUpBridge(mockRootView, mockManager, mockViewEmbedder);
ViewParent mockParent = mock(ViewParent.class);
when(mockRootView.getParent()).thenReturn(mockParent);
when(mockManager.isEnabled()).thenReturn(true);
when(mockManager.isTouchExplorationEnabled()).thenReturn(true);
TestSemanticsNode root = new TestSemanticsNode();
root.id = 0;
root.left = 0;
root.top = 0;
root.bottom = 20;
root.right = 20;
TestSemanticsNode platformView = new TestSemanticsNode();
platformView.id = 1;
platformView.platformViewId = 1;
platformView.left = 0;
platformView.top = 0;
platformView.bottom = 20;
platformView.right = 20;
root.addChild(platformView);
TestSemanticsUpdate testSemanticsUpdate = root.toUpdate();
testSemanticsUpdate.sendUpdateToBridge(accessibilityBridge);
// Synthesize an accessibility hit test event.
MotionEvent mockEvent = mock(MotionEvent.class);
when(mockEvent.getX()).thenReturn(10.0f);
when(mockEvent.getY()).thenReturn(10.0f);
when(mockEvent.getAction()).thenReturn(MotionEvent.ACTION_HOVER_ENTER);
final boolean handled = accessibilityBridge.onAccessibilityHoverEvent(mockEvent);
assertTrue(handled);
}
@Test
public void itIgnoresPlatformViewsDuringHoverIfRequested() {
AccessibilityViewEmbedder mockViewEmbedder = mock(AccessibilityViewEmbedder.class);
AccessibilityManager mockManager = mock(AccessibilityManager.class);
View mockRootView = mock(View.class);
Context context = mock(Context.class);
when(mockRootView.getContext()).thenReturn(context);
when(context.getPackageName()).thenReturn("test");
AccessibilityBridge accessibilityBridge =
setUpBridge(mockRootView, mockManager, mockViewEmbedder);
ViewParent mockParent = mock(ViewParent.class);
when(mockRootView.getParent()).thenReturn(mockParent);
when(mockManager.isEnabled()).thenReturn(true);
when(mockManager.isTouchExplorationEnabled()).thenReturn(true);
TestSemanticsNode root = new TestSemanticsNode();
root.id = 0;
root.left = 0;
root.top = 0;
root.bottom = 20;
root.right = 20;
TestSemanticsNode platformView = new TestSemanticsNode();
platformView.id = 1;
platformView.platformViewId = 1;
platformView.left = 0;
platformView.top = 0;
platformView.bottom = 20;
platformView.right = 20;
root.addChild(platformView);
TestSemanticsUpdate testSemanticsUpdate = root.toUpdate();
testSemanticsUpdate.sendUpdateToBridge(accessibilityBridge);
// Synthesize an accessibility hit test event.
MotionEvent mockEvent = mock(MotionEvent.class);
when(mockEvent.getX()).thenReturn(10.0f);
when(mockEvent.getY()).thenReturn(10.0f);
when(mockEvent.getAction()).thenReturn(MotionEvent.ACTION_HOVER_ENTER);
final boolean handled = accessibilityBridge.onAccessibilityHoverEvent(mockEvent, true);
assertFalse(handled);
}
@Test
public void itAnnouncesRouteNameWhenRemoveARoute() {
AccessibilityViewEmbedder mockViewEmbedder = mock(AccessibilityViewEmbedder.class);

View File

@@ -113,6 +113,7 @@
<src file="../../../flutter/shell/platform/android/io/flutter/plugin/platform/SingleViewPresentation.java" />
<src file="../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewFactory.java" />
<src file="../../../flutter/shell/platform/android/io/flutter/plugin/platform/AccessibilityEventsDelegate.java" />
<src file="../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformOverlayView.java" />
<src file="../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformView.java" />
<src file="../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewsAccessibilityDelegate.java" />
<src file="../../../flutter/shell/platform/android/io/flutter/plugin/platform/PlatformViewWrapper.java" />