AccessibilityBridge support for edge triggered semantics (iOS + Android) (#4901)

AccessibilityBridge support for edge triggered semantics (iOS + Android) 

lib/ui/semantics.dart

@@ -225,6 +225,8 @@ class SemanticsFlag {
   static const int _kIsInMutuallyExclusiveGroupIndex = 1 << 8;
   static const int _kIsHeaderIndex = 1 << 9;
   static const int _kIsObscuredIndex = 1 << 10;
+  static const int _kScopesRouteIndex= 1 << 11;
+  static const int _kNamesRouteIndex = 1 << 12;
 
   const SemanticsFlag._(this.index);
 
@@ -307,6 +309,44 @@ class SemanticsFlag {
   /// is a password or contains other sensitive information.
   static const SemanticsFlag isObscured = const SemanticsFlag._(_kIsObscuredIndex);
 
+  /// Whether the semantics node is the root of a subtree for which a route name
+  /// should be announced.
+  /// 
+  /// When a node with this flag is removed from the semantics tree, the 
+  /// framework will select the last in depth-first, paint order node with this
+  /// flag.  When a node with this flag is added to the semantics tree, it is
+  /// selected automatically, unless there were multiple nodes with this flag
+  /// added.  In this case, the last added node in depth-first, paint order
+  /// will be selected.
+  /// 
+  /// From this selected node, the framework will search in depth-first, paint
+  /// order for the first node with a [namesRoute] flag and a non-null, 
+  /// non-empty label. The [namesRoute] and [scopesRoute] flags may be on the 
+  /// same node. The label of the found node will be announced as an edge 
+  /// transition. If no non-empty, non-null label is found then:
+  ///   
+  ///   * VoiceOver will make a chime announcement.
+  ///   * TalkBack will make no announcement
+  ///
+  /// Semantic nodes annotated with this flag are generally not a11y focusable.
+  /// 
+  /// This is used in widgets such as Routes, Drawers, and Dialogs to 
+  /// communicate significant changes in the visible screen.
+  static const SemanticsFlag scopesRoute = const SemanticsFlag._(_kScopesRouteIndex);
+
+  /// Whether the semantics node label is the name of a visually distinct
+  /// route.
+  /// 
+  /// This is used by certain widgets like Drawers and Dialogs, to indicate
+  /// that the node's semantic label can be used to announce an edge triggered
+  /// semantics update.
+  ///
+  /// Semantic nodes annotated with this flag will still recieve a11y focus.
+  /// 
+  /// Updating this label within the same active route subtree will not cause 
+  /// additional announcements.
+  static const SemanticsFlag namesRoute = const SemanticsFlag._(_kNamesRouteIndex);
+
   /// The possible semantics flags.
   ///
   /// The map's key is the [index] of the flag and the value is the flag itself.
@@ -322,6 +362,8 @@ class SemanticsFlag {
     _kIsInMutuallyExclusiveGroupIndex: isInMutuallyExclusiveGroup,
     _kIsHeaderIndex: isHeader,
     _kIsObscuredIndex: isObscured,
+    _kScopesRouteIndex: scopesRoute,
+    _kNamesRouteIndex: namesRoute,
   };
 
   @override
@@ -349,6 +391,10 @@ class SemanticsFlag {
         return 'SemanticsFlag.isHeader';
       case _kIsObscuredIndex:
         return 'SemanticsFlag.isObscured';
+      case _kScopesRouteIndex:
+        return 'SemanticsFlag.scopesRoute';
+      case _kNamesRouteIndex:
+        return 'SemanticsFlag.namesRoute';
     }
     return null;
   }

lib/ui/semantics/semantics_node.h

@@ -55,6 +55,9 @@ enum class SemanticsFlags : int32_t {
   kIsEnabled = 1 << 7,
   kIsInMutuallyExclusiveGroup = 1 << 8,
   kIsHeader = 1 << 9,
+  kIsObscured = 1 << 10,
+  kScopesRoute = 1 << 11,
+  kNamesRoute = 1 << 12,
 };
 
 struct SemanticsNode {

shell/platform/android/io/flutter/view/AccessibilityBridge.java

@@ -38,6 +38,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
 
     private static final float SCROLL_EXTENT_FOR_INFINITY = 100000.0f;
     private static final float SCROLL_POSITION_CAP_FOR_INFINITY = 70000.0f;
+    private static final int ROOT_NODE_ID = 0;
 
     private Map<Integer, SemanticsObject> mObjects;
     private final FlutterView mOwner;
@@ -45,6 +46,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
     private SemanticsObject mA11yFocusedObject;
     private SemanticsObject mInputFocusedObject;
     private SemanticsObject mHoveredObject;
+    private int previousRouteId = ROOT_NODE_ID;
+    private List<Integer> previousRoutes;
 
     private final BasicMessageChannel<Object> mFlutterAccessibilityChannel;
 
@@ -85,7 +88,9 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         IS_ENABLED(1 << 7),
         IS_IN_MUTUALLY_EXCLUSIVE_GROUP(1 << 8),
         IS_HEADER(1 << 9),
-        IS_OBSCURED(1 << 10);
+        IS_OBSCURED(1 << 10),
+        SCOPES_ROUTE(1 << 11),
+        NAMES_ROUTE(1 << 12);
 
         Flag(int value) {
             this.value = value;
@@ -98,6 +103,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         assert owner != null;
         mOwner = owner;
         mObjects = new HashMap<Integer, SemanticsObject>();
+        previousRoutes = new ArrayList<>();
         mFlutterAccessibilityChannel = new BasicMessageChannel<>(owner, "flutter/accessibility",
             StandardMessageCodec.INSTANCE);
     }
@@ -117,8 +123,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         if (virtualViewId == View.NO_ID) {
             AccessibilityNodeInfo result = AccessibilityNodeInfo.obtain(mOwner);
             mOwner.onInitializeAccessibilityNodeInfo(result);
-            if (mObjects.containsKey(0))
-                result.addChild(mOwner, 0);
+            if (mObjects.containsKey(ROOT_NODE_ID))
+                result.addChild(mOwner, ROOT_NODE_ID);
             return result;
         }
 
@@ -177,10 +183,10 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         }
 
         if (object.parent != null) {
-            assert object.id > 0;
+            assert object.id > ROOT_NODE_ID;
             result.setParent(mOwner, object.parent.id);
         } else {
-            assert object.id == 0;
+            assert object.id == ROOT_NODE_ID;
             result.setParent(mOwner);
         }
 
@@ -479,10 +485,32 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
 
         Set<SemanticsObject> visitedObjects = new HashSet<SemanticsObject>();
         SemanticsObject rootObject = getRootObject();
+        List<SemanticsObject> newRoutes = new ArrayList<>();
         if (rootObject != null) {
           final float[] identity = new float[16];
           Matrix.setIdentityM(identity, 0);
           rootObject.updateRecursively(identity, visitedObjects, false);
+          rootObject.collectRoutes(newRoutes);
+        }
+
+        // Dispatch a TYPE_WINDOW_STATE_CHANGED event if the most recent route id changed from the
+        // previously cached route id.
+        SemanticsObject lastAdded = null;
+        for (SemanticsObject semanticsObject : newRoutes) {
+            if (!previousRoutes.contains(semanticsObject.id)) {
+                lastAdded = semanticsObject;
+            }
+        }
+        if (lastAdded == null && newRoutes.size() > 0) {
+            lastAdded = newRoutes.get(newRoutes.size() - 1);
+        }
+        if (lastAdded != null && lastAdded.id != previousRouteId) {
+            previousRouteId = lastAdded.id;
+            createWindowChangeEvent(lastAdded);
+        }
+        previousRoutes.clear();
+        for (SemanticsObject semanticsObject : newRoutes) {
+            previousRoutes.add(semanticsObject.id);
         }
 
         Iterator<Map.Entry<Integer, SemanticsObject>> it = mObjects.entrySet().iterator();
@@ -606,7 +634,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
     }
 
     private AccessibilityEvent obtainAccessibilityEvent(int virtualViewId, int eventType) {
-        assert virtualViewId != 0;
+        assert virtualViewId != ROOT_NODE_ID;
         AccessibilityEvent event = AccessibilityEvent.obtain(eventType);
         event.setPackageName(mOwner.getContext().getPackageName());
         event.setSource(mOwner, virtualViewId);
@@ -617,7 +645,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         if (!mAccessibilityEnabled) {
             return;
         }
-        if (virtualViewId == 0) {
+        if (virtualViewId == ROOT_NODE_ID) {
             mOwner.sendAccessibilityEvent(eventType);
         } else {
             sendAccessibilityEvent(obtainAccessibilityEvent(virtualViewId, eventType));
@@ -648,6 +676,13 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         }
     }
 
+    private void createWindowChangeEvent(SemanticsObject route) {
+        AccessibilityEvent e = obtainAccessibilityEvent(route.id, AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED);
+        String routeName = route.getRouteName();
+        e.getText().add(routeName);
+        mOwner.getParent().requestSendAccessibilityEvent(mOwner, e);
+    }
+
     private void willRemoveSemanticsObject(SemanticsObject object) {
         assert mObjects.containsKey(object.id);
         assert mObjects.get(object.id) == object;
@@ -875,6 +910,11 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         // TODO(goderbauer): This should be decided by the framework once we have more information
         //     about focusability there.
         boolean isFocusable() {
+            // We enforce in the framework that no other useful semantics are merged with these
+            // nodes.
+            if (hasFlag(Flag.SCOPES_ROUTE)) {
+                return false;
+            }
             int scrollableActions = Action.SCROLL_RIGHT.value | Action.SCROLL_LEFT.value
                     | Action.SCROLL_UP.value | Action.SCROLL_DOWN.value;
             return (actions & ~scrollableActions) != 0
@@ -884,6 +924,36 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
                 || (hint != null && !hint.isEmpty());
         }
 
+        void collectRoutes(List<SemanticsObject> edges) {
+            if (hasFlag(Flag.SCOPES_ROUTE)) {
+                edges.add(this);
+            }
+            if (children != null) {
+                for (int i = 0; i < children.size(); ++i) {
+                    children.get(i).collectRoutes(edges);
+                }
+            }
+        }
+
+        String getRouteName() {
+            // Returns the first non-null and non-empty semantic label of a child
+            // with an NamesRoute flag. Otherwise returns null.
+            if (hasFlag(Flag.NAMES_ROUTE)) {
+                if (label != null && !label.isEmpty()) {
+                    return label;
+                }
+            }
+            if (children != null) {
+                for (int i = 0; i < children.size(); ++i) {
+                    String newName = children.get(i).getRouteName();
+                    if (newName != null && !newName.isEmpty()) {
+                        return newName;
+                    }
+                }
+            }
+            return null;
+        }
+
         void updateRecursively(float[] ancestorTransform, Set<SemanticsObject> visitedObjects, boolean forceUpdate) {
             visitedObjects.add(this);
 

shell/platform/darwin/ios/framework/Source/accessibility_bridge.h

@@ -122,6 +122,8 @@ class AccessibilityBridge final {
   fml::scoped_nsobject<NSMutableDictionary<NSNumber*, SemanticsObject*>> objects_;
   fml::scoped_nsprotocol<FlutterBasicMessageChannel*> accessibility_channel_;
   fml::WeakPtrFactory<AccessibilityBridge> weak_factory_;
+  int32_t previous_route_id_;
+  std::vector<int32_t> previous_routes_;
 
   FXL_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge);
 };

shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm

@@ -164,11 +164,45 @@ NSComparisonResult IntToComparisonResult(int32_t value) {
   // Note: hit detection will only apply to elements that report
   // -isAccessibilityElement of YES. The framework will continue scanning the
   // entire element tree looking for such a hit.
+
+  //  We enforce in the framework that no other useful semantics are merged with these nodes.
+  if ([self node].HasFlag(blink::SemanticsFlags::kScopesRoute))
+    return false;
   return [self node].flags != 0 || ![self node].label.empty() || ![self node].value.empty() ||
          ![self node].hint.empty() ||
          ([self node].actions & ~blink::kScrollableSemanticsActions) != 0;
 }
 
+- (void)collectRoutes:(NSMutableArray<SemanticsObject*>*)edges {
+  if ([self node].HasFlag(blink::SemanticsFlags::kScopesRoute))
+    [edges addObject:self];
+  if ([self hasChildren]) {
+    for (SemanticsObject* child in self.children) {
+      [child collectRoutes:edges];
+    }
+  }
+}
+
+- (NSString*)routeName {
+  // Returns the first non-null and non-empty semantic label of a child
+  // with an NamesRoute flag. Otherwise returns nil.
+  if ([self node].HasFlag(blink::SemanticsFlags::kNamesRoute)) {
+    NSString* newName = [self accessibilityLabel];
+    if (newName != nil && [newName length] > 0) {
+      return newName;
+    }
+  }
+  if ([self hasChildren]) {
+    for (SemanticsObject* child in self.children) {
+      NSString* newName = [child routeName];
+      if (newName != nil && [newName length] > 0) {
+        return newName;
+      }
+    }
+  }
+  return nil;
+}
+
 - (NSString*)accessibilityLabel {
   if ([self node].label.empty())
     return nil;
@@ -424,7 +458,9 @@ AccessibilityBridge::AccessibilityBridge(UIView* view, PlatformViewIOS* platform
     : view_(view),
       platform_view_(platform_view),
       objects_([[NSMutableDictionary alloc] init]),
-      weak_factory_(this) {
+      weak_factory_(this),
+      previous_route_id_(0),
+      previous_routes_({}) {
   accessibility_channel_.reset([[FlutterBasicMessageChannel alloc]
          initWithName:@"flutter/accessibility"
       binaryMessenger:platform_view->GetOwnerViewController()
@@ -492,10 +528,32 @@ void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes) {
 
   SemanticsObject* root = objects_.get()[@(kRootNodeId)];
 
+  bool routeChanged = false;
+  SemanticsObject* lastAdded = nil;
+
   if (root) {
     if (!view_.accessibilityElements) {
       view_.accessibilityElements = @[ [root accessibilityContainer] ];
     }
+    NSMutableArray<SemanticsObject*>* newRoutes = [[[NSMutableArray alloc] init] autorelease];
+    [root collectRoutes:newRoutes];
+    for (SemanticsObject* route in newRoutes) {
+      if (std::find(previous_routes_.begin(), previous_routes_.end(), [route uid]) != previous_routes_.end()) {
+        lastAdded = route;
+      }
+    }
+    if (lastAdded == nil && [newRoutes count] > 0) {
+      int index = [newRoutes count] - 1;
+      lastAdded = [newRoutes objectAtIndex:index];
+    }
+    if (lastAdded != nil && [lastAdded uid] != previous_route_id_) {
+        previous_route_id_ = [lastAdded uid];
+        routeChanged = true;
+    }
+    previous_routes_.clear();
+    for (SemanticsObject* route in newRoutes) {
+      previous_routes_.push_back([route uid]);
+    }
   } else {
     view_.accessibilityElements = nil;
   }
@@ -507,7 +565,10 @@ void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes) {
 
   layoutChanged = layoutChanged || [doomed_uids count] > 0;
 
-  if (layoutChanged) {
+  if (routeChanged) {
+    NSString* routeName = [lastAdded routeName];
+    UIAccessibilityPostNotification(UIAccessibilityScreenChangedNotification, routeName);
+  } else if (layoutChanged) {
     // TODO(goderbauer): figure out which node to focus next.
     UIAccessibilityPostNotification(UIAccessibilityLayoutChangedNotification, nil);
   }