From 99810261735aa9f7f2bd4d3105d3eba86a207d0f Mon Sep 17 00:00:00 2001 From: Emmanuel Garcia Date: Tue, 13 Oct 2020 17:52:02 -0700 Subject: [PATCH] Allow TalkBack navigation while a platform view is rendered (#21719) --- .../io/flutter/view/AccessibilityBridge.java | 40 +++++++-- .../flutter/view/AccessibilityBridgeTest.java | 84 +++++++++++++++++++ 2 files changed, 118 insertions(+), 6 deletions(-) diff --git a/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/shell/platform/android/io/flutter/view/AccessibilityBridge.java index cd5089451..92cbe68b1 100644 --- a/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -28,6 +28,7 @@ import androidx.annotation.RequiresApi; import androidx.annotation.VisibleForTesting; import io.flutter.BuildConfig; import io.flutter.Log; +import io.flutter.embedding.android.FlutterActivity; import io.flutter.embedding.engine.systemchannels.AccessibilityChannel; import io.flutter.plugin.platform.PlatformViewsAccessibilityDelegate; import io.flutter.util.Predicate; @@ -541,12 +542,22 @@ public class AccessibilityBridge extends AccessibilityNodeProvider { return null; } + // Generate accessibility node for platform views using a virtual display. + // + // In this case, register the accessibility node in the view embedder, + // so the accessibility tree can be mirrored as a subtree of the Flutter accessibility tree. + // This is in constrast to hybrid composition where the embeded view is in the view hiearchy, + // so it doesn't need to be mirrored. + // + // See the case down below for how hybrid composition is handled. if (semanticsNode.platformViewId != -1) { - // For platform views we delegate the node creation to the accessibility view embedder. View embeddedView = platformViewsAccessibilityDelegate.getPlatformViewById(semanticsNode.platformViewId); - Rect bounds = semanticsNode.getGlobalRect(); - return accessibilityViewEmbedder.getRootNode(embeddedView, semanticsNode.id, bounds); + boolean childUsesVirtualDisplay = !(embeddedView.getContext() instanceof FlutterActivity); + if (childUsesVirtualDisplay) { + Rect bounds = semanticsNode.getGlobalRect(); + return accessibilityViewEmbedder.getRootNode(embeddedView, semanticsNode.id, bounds); + } } AccessibilityNodeInfo result = @@ -823,11 +834,28 @@ public class AccessibilityBridge extends AccessibilityNodeProvider { } for (SemanticsNode child : semanticsNode.childrenInTraversalOrder) { - if (!child.hasFlag(Flag.IS_HIDDEN)) { - result.addChild(rootAccessibilityView, child.id); + if (child.hasFlag(Flag.IS_HIDDEN)) { + continue; + } + if (child.platformViewId != -1) { + View embeddedView = + platformViewsAccessibilityDelegate.getPlatformViewById(child.platformViewId); + + // Add the embeded view as a child of the current accessibility node if it's using + // hybrid composition. + // + // In this case, the view is in the Activity's view hierarchy, so it doesn't need to be + // mirrored. + // + // See the case above for how virtual displays are handled. + boolean childUsesHybridComposition = embeddedView.getContext() instanceof FlutterActivity; + if (childUsesHybridComposition) { + result.addChild(embeddedView); + continue; + } } + result.addChild(rootAccessibilityView, child.id); } - return result; } diff --git a/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java b/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java index 81d87ae56..673c1e48c 100644 --- a/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java +++ b/shell/platform/android/test/io/flutter/view/AccessibilityBridgeTest.java @@ -5,6 +5,7 @@ package io.flutter.view; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyInt; @@ -16,14 +17,17 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import android.annotation.TargetApi; +import android.app.Activity; import android.content.ContentResolver; import android.content.Context; +import android.graphics.Rect; import android.view.MotionEvent; import android.view.View; import android.view.ViewParent; import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityManager; import android.view.accessibility.AccessibilityNodeInfo; +import io.flutter.embedding.android.FlutterActivity; import io.flutter.embedding.engine.systemchannels.AccessibilityChannel; import io.flutter.plugin.platform.PlatformViewsAccessibilityDelegate; import java.nio.ByteBuffer; @@ -33,6 +37,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.robolectric.RobolectricTestRunner; +import org.robolectric.RuntimeEnvironment; import org.robolectric.annotation.Config; @Config(manifest = Config.NONE) @@ -198,6 +203,81 @@ public class AccessibilityBridgeTest { accessibilityBridge.onAccessibilityHoverEvent(MotionEvent.obtain(1, 1, 1, -10, -10, 0)); } + @Test + public void itProducesPlatformViewNodeForHybridComposition() { + PlatformViewsAccessibilityDelegate accessibilityDelegate = + mock(PlatformViewsAccessibilityDelegate.class); + + Context context = RuntimeEnvironment.application.getApplicationContext(); + View rootAccessibilityView = new View(context); + AccessibilityViewEmbedder accessibilityViewEmbedder = mock(AccessibilityViewEmbedder.class); + AccessibilityBridge accessibilityBridge = + setUpBridge( + rootAccessibilityView, + /*accessibilityChannel=*/ null, + /*accessibilityManager=*/ null, + /*contentResolver=*/ null, + accessibilityViewEmbedder, + accessibilityDelegate); + + TestSemanticsNode root = new TestSemanticsNode(); + root.id = 0; + + TestSemanticsNode platformView = new TestSemanticsNode(); + platformView.id = 1; + platformView.platformViewId = 1; + root.addChild(platformView); + + TestSemanticsUpdate testSemanticsRootUpdate = root.toUpdate(); + accessibilityBridge.updateSemantics( + testSemanticsRootUpdate.buffer, testSemanticsRootUpdate.strings); + + TestSemanticsUpdate testSemanticsPlatformViewUpdate = platformView.toUpdate(); + accessibilityBridge.updateSemantics( + testSemanticsPlatformViewUpdate.buffer, testSemanticsPlatformViewUpdate.strings); + + View embeddedView = mock(View.class); + when(accessibilityDelegate.getPlatformViewById(1)).thenReturn(embeddedView); + + when(embeddedView.getContext()).thenReturn(mock(FlutterActivity.class)); + + AccessibilityNodeInfo nodeInfo = mock(AccessibilityNodeInfo.class); + when(embeddedView.createAccessibilityNodeInfo()).thenReturn(nodeInfo); + + AccessibilityNodeInfo result = accessibilityBridge.createAccessibilityNodeInfo(0); + assertNotNull(result); + assertEquals(result.getChildCount(), 1); + assertEquals(result.getClassName(), "android.view.View"); + } + + @Test + public void itProducesPlatformViewNodeForVirtualDisplay() { + PlatformViewsAccessibilityDelegate accessibilityDelegate = + mock(PlatformViewsAccessibilityDelegate.class); + AccessibilityViewEmbedder accessibilityViewEmbedder = mock(AccessibilityViewEmbedder.class); + AccessibilityBridge accessibilityBridge = + setUpBridge( + /*rootAccessibilityView=*/ null, + /*accessibilityChannel=*/ null, + /*accessibilityManager=*/ null, + /*contentResolver=*/ null, + accessibilityViewEmbedder, + accessibilityDelegate); + + TestSemanticsNode platformView = new TestSemanticsNode(); + platformView.platformViewId = 1; + + TestSemanticsUpdate testSemanticsUpdate = platformView.toUpdate(); + accessibilityBridge.updateSemantics(testSemanticsUpdate.buffer, testSemanticsUpdate.strings); + + View embeddedView = mock(View.class); + when(accessibilityDelegate.getPlatformViewById(1)).thenReturn(embeddedView); + when(embeddedView.getContext()).thenReturn(mock(Activity.class)); + + accessibilityBridge.createAccessibilityNodeInfo(0); + verify(accessibilityViewEmbedder).getRootNode(eq(embeddedView), eq(0), any(Rect.class)); + } + @Test public void releaseDropsChannelMessageHandler() { AccessibilityChannel mockChannel = mock(AccessibilityChannel.class); @@ -317,6 +397,10 @@ public class AccessibilityBridgeTest { float right = 0.0f; float bottom = 0.0f; final List children = new ArrayList(); + + public void addChild(TestSemanticsNode child) { + children.add(child); + } // custom actions not supported. TestSemanticsUpdate toUpdate() { -- GitLab