Adding semantics traversal order support (#4540)

This adds support for semantics traversal ordering.

It is a companion to flutter/flutter#14060, adding support for a sortIndex in the semantics data passed to the engine.

Addresses flutter/flutter#12187

lib/ui/semantics.dart

@@ -340,7 +340,7 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
   /// The `rect` is the region occupied by this node in its own coordinate
   /// system.
   ///
-  /// The `transform` is a matrix that maps this node's coodinate system into
+  /// The `transform` is a matrix that maps this node's coordinate system into
   /// its parent's coordinate system.
   void updateNode({
     int id,
@@ -355,30 +355,30 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
     String increasedValue,
     String decreasedValue,
     TextDirection textDirection,
+    int nextNodeId,
     Float64List transform,
     Int32List children,
   }) {
     if (transform.length != 16)
       throw new ArgumentError('transform argument must have 16 entries.');
-    _updateNode(
-      id,
-      flags,
-      actions,
-      textSelectionBase,
-      textSelectionExtent,
-      rect.left,
-      rect.top,
-      rect.right,
-      rect.bottom,
-      label,
-      hint,
-      value,
-      increasedValue,
-      decreasedValue,
-      textDirection != null ? textDirection.index + 1 : 0,
-      transform,
-      children,
-    );
+    _updateNode(id,
+                flags,
+                actions,
+                textSelectionBase,
+                textSelectionExtent,
+                rect.left,
+                rect.top,
+                rect.right,
+                rect.bottom,
+                label,
+                hint,
+                value,
+                increasedValue,
+                decreasedValue,
+                textDirection != null ? textDirection.index + 1 : 0,
+                nextNodeId ?? -1,
+                transform,
+                children,);
   }
   void _updateNode(
     int id,
@@ -396,6 +396,7 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
     String increasedValue,
     String decreasedValue,
     int textDirection,
+    int nextNodeId,
     Float64List transform,
     Int32List children,
   ) native 'SemanticsUpdateBuilder_updateNode';

lib/ui/semantics/semantics_node.h

@@ -63,6 +63,7 @@ struct SemanticsNode {
   std::string increasedValue;
   std::string decreasedValue;
   int32_t textDirection = 0;  // 0=unknown, 1=rtl, 2=ltr
+  int32_t nextNodeId = -1;
 
   SkRect rect = SkRect::MakeEmpty();
   SkMatrix44 transform = SkMatrix44(SkMatrix44::kIdentity_Constructor);

lib/ui/semantics/semantics_update_builder.cc

@@ -49,6 +49,7 @@ void SemanticsUpdateBuilder::updateNode(int id,
                                         std::string increasedValue,
                                         std::string decreasedValue,
                                         int textDirection,
+                                        int nextNodeId,
                                         const tonic::Float64List& transform,
                                         const tonic::Int32List& children) {
   SemanticsNode node;
@@ -64,6 +65,7 @@ void SemanticsUpdateBuilder::updateNode(int id,
   node.increasedValue = increasedValue;
   node.decreasedValue = decreasedValue;
   node.textDirection = textDirection;
+  node.nextNodeId = nextNodeId;
   node.transform.setColMajord(transform.data());
   node.children = std::vector<int32_t>(
       children.data(), children.data() + children.num_elements());

lib/ui/semantics/semantics_update_builder.h

@@ -40,6 +40,7 @@ class SemanticsUpdateBuilder
                   std::string increasedValue,
                   std::string decreasedValue,
                   int textDirection,
+                  int nextNodeId,
                   const tonic::Float64List& transform,
                   const tonic::Int32List& children);
 

lib/ui/window.dart

@@ -342,7 +342,7 @@ class Window {
   ///
   /// When this changes, [onMetricsChanged] is called.
   ///
-  /// At startup, the size of the application window may not be know before Dart
+  /// At startup, the size of the application window may not be known before Dart
   /// code runs. If this value is observed early in the application lifecycle,
   /// it may report [Size.zero].
   ///

shell/platform/android/io/flutter/view/AccessibilityBridge.java

@@ -236,6 +236,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
 
         result.setSelected(object.hasFlag(Flag.IS_SELECTED));
         result.setText(object.getValueLabelHint());
+        result.setTraversalBefore(mOwner,
+            object.nextNodeId == -1 ? View.NO_ID : object.nextNodeId);
 
         // Accessibility Focus
         if (mA11yFocusedObject != null && mA11yFocusedObject.id == virtualViewId) {
@@ -245,16 +247,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         }
 
         if (object.children != null) {
-            List<SemanticsObject> childrenInTraversalOrder =
-                new ArrayList<SemanticsObject>(object.children);
-            Collections.sort(childrenInTraversalOrder, new Comparator<SemanticsObject>() {
-                public int compare(SemanticsObject a, SemanticsObject b) {
-                    final int top = Integer.compare(a.globalRect.top, b.globalRect.top);
-                    // TODO(goderbauer): sort right-to-left in rtl environments.
-                    return top == 0 ? Integer.compare(a.globalRect.left, b.globalRect.left) : top;
-                }
-            });
-            for (SemanticsObject child : childrenInTraversalOrder) {
+            for (SemanticsObject child : object.children) {
                 result.addChild(mOwner, child.id);
             }
         }
@@ -673,6 +666,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         String decreasedValue;
         String hint;
         TextDirection textDirection;
+        int nextNodeId;
 
         boolean hadPreviousConfig = false;
         int previousFlags;
@@ -709,14 +703,16 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
             return (previousFlags & flag.value) != 0;
         }
 
-        void log(String indent) {
+        void log(String indent, boolean recursive) {
           Log.i(TAG, indent + "SemanticsObject id=" + id + " label=" + label + " actions=" +  actions + " flags=" + flags + "\n" +
+                     indent + "  +-- textDirection=" + textDirection  + "\n"+
+                     indent + "  +-- nextNodeId=" + nextNodeId  + "\n"+
                      indent + "  +-- rect.ltrb=(" + left + ", " + top + ", " + right + ", " + bottom + ")\n" +
                      indent + "  +-- transform=" + Arrays.toString(transform) + "\n");
-          if (children != null) {
+          if (children != null && recursive) {
               String childIndent = indent + "  ";
               for (SemanticsObject child : children) {
-                  child.log(childIndent);
+                  child.log(childIndent, recursive);
               }
           }
         }
@@ -750,6 +746,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
 
             textDirection = TextDirection.fromInt(buffer.getInt());
 
+            nextNodeId = buffer.getInt();
+
             left = buffer.getFloat();
             top = buffer.getFloat();
             right = buffer.getFloat();

shell/platform/android/platform_view_android.cc

@@ -461,7 +461,7 @@ bool PlatformViewAndroid::ResourceContextMakeCurrent() {
 
 void PlatformViewAndroid::UpdateSemantics(
     std::vector<blink::SemanticsNode> update) {
-  constexpr size_t kBytesPerNode = 32 * sizeof(int32_t);
+  constexpr size_t kBytesPerNode = 33 * sizeof(int32_t);
   constexpr size_t kBytesPerChild = sizeof(int32_t);
 
   JNIEnv* env = fml::jni::AttachCurrentThread();
@@ -484,7 +484,8 @@ void PlatformViewAndroid::UpdateSemantics(
     size_t position = 0;
     for (const blink::SemanticsNode& node : update) {
       // If you edit this code, make sure you update kBytesPerNode
-      // above to match the number of values you are sending.
+      // and/or kBytesPerChild above to match the number of values you are
+      // sending.
       buffer_int32[position++] = node.id;
       buffer_int32[position++] = node.flags;
       buffer_int32[position++] = node.actions;
@@ -521,6 +522,7 @@ void PlatformViewAndroid::UpdateSemantics(
         strings.push_back(node.hint);
       }
       buffer_int32[position++] = node.textDirection;
+      buffer_int32[position++] = node.nextNodeId;
       buffer_float32[position++] = node.rect.left();
       buffer_float32[position++] = node.rect.top();
       buffer_float32[position++] = node.rect.right();