Merge pull request #2353 from Hixie/actions

Make AccessibilityNodeInfos interactive.

sky/shell/platform/android/org/domokit/sky/shell/FlutterSemanticsToAndroidAccessibilityBridge.java

@@ -6,7 +6,9 @@ package org.domokit.sky.shell;
 
 import android.graphics.Rect;
 import android.opengl.Matrix;
+import android.os.Bundle;
 import android.view.View;
+import android.view.accessibility.AccessibilityEvent;
 import android.view.accessibility.AccessibilityManager;
 import android.view.accessibility.AccessibilityNodeInfo;
 import android.view.accessibility.AccessibilityNodeProvider;
@@ -26,10 +28,17 @@ public class FlutterSemanticsToAndroidAccessibilityBridge extends AccessibilityN
                                                           implements SemanticsListener {
     private Map<Integer, PersistentAccessibilityNode> mTreeNodes;
     private PlatformViewAndroid mOwner;
-
-    FlutterSemanticsToAndroidAccessibilityBridge(PlatformViewAndroid view) {
-        mOwner = view;
+    private SemanticsServer.Proxy mSemanticsServer;
+    private PersistentAccessibilityNode mFocusedNode;
+    private PersistentAccessibilityNode mHoveredNode;
+
+    FlutterSemanticsToAndroidAccessibilityBridge(PlatformViewAndroid owner, SemanticsServer.Proxy semanticsServer) {
+        assert owner != null;
+        assert semanticsServer != null;
+        mOwner = owner;
         mTreeNodes = new HashMap<Integer, PersistentAccessibilityNode>();
+        mSemanticsServer = semanticsServer;
+        mSemanticsServer.addSemanticsListener(this);
     }
 
     @Override
@@ -71,17 +80,46 @@ public class FlutterSemanticsToAndroidAccessibilityBridge extends AccessibilityN
         }
         result.setBoundsInScreen(bounds);
         result.setVisibleToUser(true);
+        result.setEnabled(true); // TODO(ianh): Expose disabled subtrees
 
-        // TODO(ianh): Add support for interactivity:
-        // private boolean canBeTapped;
-        // private boolean canBeLongPressed;
-        // private boolean canBeScrolledHorizontally;
-        // private boolean canBeScrolledVertically;
+        if (node.canBeTapped) {
+            result.addAction(AccessibilityNodeInfo.ACTION_CLICK);
+            result.setClickable(true);
+        }
+        if (node.canBeLongPressed) {
+            result.addAction(AccessibilityNodeInfo.ACTION_LONG_CLICK);
+            result.setLongClickable(true);
+        }
+        if ((node.canBeScrolledHorizontally && !node.canBeScrolledVertically) ||
+            (!node.canBeScrolledHorizontally && node.canBeScrolledVertically)) {
+            result.addAction(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD);
+            result.addAction(AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD);
+        }
+        if (node.canBeScrolledHorizontally || node.canBeScrolledVertically) {
+            // TODO(ianh): Figure out how to enable panning. SDK v23
+            // has AccessibilityAction.ACTION_SCROLL_LEFT and company,
+            // but earlier versions do not. Right now we only forward
+            // scroll actions if it's unidirectional.
+            result.setScrollable(true);
+        }
 
         result.setCheckable(node.hasCheckedState);
         result.setChecked(node.isChecked);
         result.setText(node.label);
 
+        // TODO(ianh): use setTraversalBefore/setTraversalAfter to set
+        // the relative order of the views. For each set of siblings,
+        // the views should be ordered top-to-bottom, tie-breaking
+        // left-to-right (right-to-left in rtl environments), height,
+        // width, and finally by list order.
+
+        // Accessibility Focus
+        if (mFocusedNode != null && mFocusedNode.id == virtualViewId) {
+            result.addAction(AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS);
+        } else {
+            result.addAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS);
+        }
+
         for (PersistentAccessibilityNode child : node.children) {
             result.addChild(mOwner, child.id);
         }
@@ -89,10 +127,99 @@ public class FlutterSemanticsToAndroidAccessibilityBridge extends AccessibilityN
         return result;
     }
 
+    @Override
+    public boolean performAction(int virtualViewId, int action, Bundle arguments) {
+        PersistentAccessibilityNode node = mTreeNodes.get(virtualViewId);
+        if (node == null)
+            return false;
+        switch (action) {
+            case AccessibilityNodeInfo.ACTION_CLICK: {
+                mSemanticsServer.tap(virtualViewId);
+                return true;
+            }
+            case AccessibilityNodeInfo.ACTION_LONG_CLICK: {
+                mSemanticsServer.longPress(virtualViewId);
+                return true;
+            }
+            case AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD: {
+                if (node.canBeScrolledHorizontally && !node.canBeScrolledVertically) {
+                    // TODO(ianh): bidi support
+                    mSemanticsServer.scrollLeft(virtualViewId);
+                } else if (node.canBeScrolledHorizontally && !node.canBeScrolledVertically) {
+                    mSemanticsServer.scrollUp(virtualViewId);
+                } else {
+                    return false;
+                }
+                return true;
+            }
+            case AccessibilityNodeInfo.ACTION_SCROLL_FORWARD: {
+                if (node.canBeScrolledHorizontally && !node.canBeScrolledVertically) {
+                    // TODO(ianh): bidi support
+                    mSemanticsServer.scrollRight(virtualViewId);
+                } else if (node.canBeScrolledHorizontally && !node.canBeScrolledVertically) {
+                    mSemanticsServer.scrollDown(virtualViewId);
+                } else {
+                    return false;
+                }
+                return true;
+            }
+            case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS: {
+                sendAccessibilityEvent(virtualViewId, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
+                mFocusedNode = null;
+                return true;
+            }
+            case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS: {
+                sendAccessibilityEvent(virtualViewId, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED);
+                mFocusedNode = node;
+                return true;
+            }
+        }
+        // TODO(ianh): Implement left/right/up/down scrolling
+        return false;
+    }
+
+    // TODO(ianh): implement findAccessibilityNodeInfosByText()
+    // TODO(ianh): implement findFocus()
+
+    public void handleTouchExplorationExit() {
+        if (mHoveredNode != null) {
+            sendAccessibilityEvent(mHoveredNode.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT);
+            mHoveredNode = null;
+        }
+    }
+
+    public void handleTouchExploration(float x, float y) {
+        if (mTreeNodes.isEmpty())
+            return;
+        assert mTreeNodes.containsKey(0);
+        PersistentAccessibilityNode newNode = mTreeNodes.get(0).hitTest(Math.round(x), Math.round(y));
+        if (newNode != mHoveredNode) {
+            if (newNode != null) {
+                sendAccessibilityEvent(newNode.id, AccessibilityEvent.TYPE_VIEW_HOVER_ENTER);
+            }
+            if (mHoveredNode != null) {
+                sendAccessibilityEvent(mHoveredNode.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT);
+            }
+            mHoveredNode = newNode;
+        }
+    }
+
     @Override
     public void updateSemanticsTree(SemanticsNode[] nodes) {
         for (SemanticsNode node : nodes) {
             updateSemanticsNode(node);
+            sendAccessibilityEvent(node.id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
+        }
+    }
+
+    private void sendAccessibilityEvent(int virtualViewId, int eventType) {
+        if (virtualViewId == 0) {
+            mOwner.sendAccessibilityEvent(eventType);
+        } else {
+            AccessibilityEvent event = AccessibilityEvent.obtain(eventType);
+            event.setPackageName(mOwner.getContext().getPackageName());
+            event.setSource(mOwner, virtualViewId);
+            mOwner.getParent().requestSendAccessibilityEvent(mOwner, event);
         }
     }
 
@@ -112,10 +239,25 @@ public class FlutterSemanticsToAndroidAccessibilityBridge extends AccessibilityN
         assert mTreeNodes.containsKey(node.id);
         assert mTreeNodes.get(node.id).parent == null;
         mTreeNodes.remove(node.id);
+        if (mFocusedNode == node) {
+            mFocusedNode = null;
+        }
+        if (mHoveredNode == node) {
+            mHoveredNode = null;
+        }
+        for (PersistentAccessibilityNode child : node.children) {
+            removePersistentNode(child);
+        }
     }
 
-    public void reset() {
+    public void reset(SemanticsServer.Proxy newSemanticsServer) {
         mTreeNodes.clear();
+        mFocusedNode = null;
+        mHoveredNode = null;
+        mSemanticsServer.close();
+        sendAccessibilityEvent(0, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
+        mSemanticsServer = newSemanticsServer;
+        mSemanticsServer.addSemanticsListener(this);
     }
 
     private class PersistentAccessibilityNode {
@@ -177,7 +319,6 @@ public class FlutterSemanticsToAndroidAccessibilityBridge extends AccessibilityN
                 // since they also get marked dirty
                 invalidateGlobalGeometry();
             }
-            // TODO(ianh): Notify Android that our tree is dirty
         }
 
         // fields that we pass straight to the Android accessibility API
@@ -205,6 +346,8 @@ public class FlutterSemanticsToAndroidAccessibilityBridge extends AccessibilityN
                 return;
             }
             geometryDirty = true;
+            // TODO(ianh): if we are the FlutterSemanticsToAndroidAccessibilityBridge.this.mFocusedNode
+            // then we may have to unfocus and refocus ourselves to get Android to update the focus rect
             for (PersistentAccessibilityNode child : children) {
                 child.invalidateGlobalGeometry();
             }
@@ -268,6 +411,20 @@ public class FlutterSemanticsToAndroidAccessibilityBridge extends AccessibilityN
             }
             return globalRect;
         }
+
+        public PersistentAccessibilityNode hitTest(int x, int y) {
+            Rect rect = getGlobalRect();
+            if (!rect.contains(x, y))
+                return null;
+            for (int index = children.size()-1; index >= 0; index -= 1) {
+                PersistentAccessibilityNode child = children.get(index);
+                PersistentAccessibilityNode result = child.hitTest(x, y);
+                if (result != null) {
+                    return result;
+                }
+            }
+            return this;
+        }
     }
 
     @Override

sky/shell/platform/android/org/domokit/sky/shell/PlatformViewAndroid.java

@@ -37,8 +37,6 @@ import org.chromium.mojom.pointer.PointerKind;
 import org.chromium.mojom.pointer.PointerPacket;
 import org.chromium.mojom.pointer.PointerType;
 import org.chromium.mojom.raw_keyboard.RawKeyboardService;
-import org.chromium.mojom.semantics.SemanticsListener;
-import org.chromium.mojom.semantics.SemanticsNode;
 import org.chromium.mojom.semantics.SemanticsServer;
 import org.chromium.mojom.sky.ServicesData;
 import org.chromium.mojom.sky.SkyEngine;
@@ -250,6 +248,16 @@ public class PlatformViewAndroid extends SurfaceView
         return true;
     }
 
+    @Override
+    public boolean onHoverEvent(MotionEvent event) {
+        boolean handled = handleAccessibilityHoverEvent(event);
+        if (!handled) {
+            // TODO(ianh): Expose hover events to the platform,
+            // implementing ADD, REMOVE, etc.
+        }
+        return handled;
+    }
+
     @Override
     protected void onSizeChanged(int width, int height, int oldWidth, int oldHeight) {
         mMetrics.physicalWidth = width;
@@ -298,6 +306,14 @@ public class PlatformViewAndroid extends SurfaceView
     }
 
     void runFromBundle(String path) {
+
+        if (mServiceProvider != null) {
+            mServiceProvider.close();
+        }
+        if (mDartServiceProvider != null) {
+            mDartServiceProvider.close();
+        }
+
         Core core = CoreImpl.getInstance();
         Pair<ServiceProvider.Proxy, InterfaceRequest<ServiceProvider>> serviceProvider =
                 ServiceProvider.MANAGER.getInterfaceRequest(core);
@@ -326,10 +342,13 @@ public class PlatformViewAndroid extends SurfaceView
 
     // ACCESSIBILITY
 
+    private boolean mTouchExplorationEnabled = false;
+
     @Override
     protected void onAttachedToWindow() {
         super.onAttachedToWindow();
-        if (mAccessibilityManager.isEnabled() || mAccessibilityManager.isTouchExplorationEnabled())
+        mTouchExplorationEnabled = mAccessibilityManager.isTouchExplorationEnabled();
+        if (mAccessibilityManager.isEnabled() || mTouchExplorationEnabled)
           ensureAccessibilityEnabled();
         mAccessibilityManager.addAccessibilityStateChangeListener(this);
         mAccessibilityManager.addTouchExplorationStateChangeListener(this);
@@ -343,25 +362,15 @@ public class PlatformViewAndroid extends SurfaceView
 
     @Override
     public void onTouchExplorationStateChanged(boolean enabled) {
-        if (enabled)
+        if (enabled) {
+            mTouchExplorationEnabled = true;
             ensureAccessibilityEnabled();
-        // TODO(ianh): else, actually discard the state for exploration
-    }
-
-    private FlutterSemanticsToAndroidAccessibilityBridge mAccessibilityNodeProvider;
-    private SemanticsServer.Proxy mSemanticsServer;
-
-    void ensureAccessibilityEnabled() {
-        if (mAccessibilityNodeProvider == null) {
-            mAccessibilityNodeProvider = new FlutterSemanticsToAndroidAccessibilityBridge(this);
-            Core core = CoreImpl.getInstance();
-            Pair<SemanticsServer.Proxy, InterfaceRequest<SemanticsServer>> server =
-                      SemanticsServer.MANAGER.getInterfaceRequest(core);
-            mSemanticsServer = server.first;
-            mDartServiceProvider.connectToService(SemanticsServer.MANAGER.getName(), server.second.passHandle());
-            mSemanticsServer.addSemanticsListener(mAccessibilityNodeProvider);
+        } else {
+            mTouchExplorationEnabled = false;
+            if (mAccessibilityNodeProvider != null) {
+                mAccessibilityNodeProvider.handleTouchExplorationExit();
+            }
         }
-        assert mSemanticsServer != null;
     }
 
     @Override
@@ -370,13 +379,37 @@ public class PlatformViewAndroid extends SurfaceView
         return mAccessibilityNodeProvider;
     }
 
-    // TODO(ianh): implement touch exploration
+    private FlutterSemanticsToAndroidAccessibilityBridge mAccessibilityNodeProvider;
+
+    void ensureAccessibilityEnabled() {
+        if (mAccessibilityNodeProvider == null) {
+            mAccessibilityNodeProvider = new FlutterSemanticsToAndroidAccessibilityBridge(this, createSemanticsServer());
+        }
+    }
 
-    // TODO(ianh): implement accessibility focus
+    private SemanticsServer.Proxy createSemanticsServer() {
+        Core core = CoreImpl.getInstance();
+        Pair<SemanticsServer.Proxy, InterfaceRequest<SemanticsServer>> server =
+                  SemanticsServer.MANAGER.getInterfaceRequest(core);
+        mDartServiceProvider.connectToService(SemanticsServer.MANAGER.getName(), server.second.passHandle());
+        return server.first;
+    }
 
     void resetAccessibilityTree() {
-        if (mAccessibilityNodeProvider != null)
-            mAccessibilityNodeProvider.reset();
+        if (mAccessibilityNodeProvider != null) {
+            mAccessibilityNodeProvider.reset(createSemanticsServer());
+        }
+    }
+
+    private boolean handleAccessibilityHoverEvent(MotionEvent event) {
+        if (!mTouchExplorationEnabled)
+            return false;
+        if (event.getAction() == MotionEvent.ACTION_HOVER_EXIT) {
+            mAccessibilityNodeProvider.handleTouchExplorationExit();
+        } else {
+            mAccessibilityNodeProvider.handleTouchExploration(event.getX(), event.getY());
+        }
+        return true;
     }
 
 }