Encode scrolling status into tree (#4647)

lib/ui/semantics.dart

@@ -337,6 +337,13 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
   /// The fields 'textSelectionBase' and 'textSelectionExtent' describe the
   /// currently selected text within `value`.
   ///
+  /// For scrollable nodes `scrollPosition` describes the current scroll
+  /// position in logical pixel. `scrollExtentMax` and `scrollExtentMin`
+  /// describe the maximum and minimum in-rage values that `scrollPosition` can
+  /// be. Both or either may be infinity to indicate unbound scrolling. The
+  /// value for `scrollPosition` can (temporarily) be outside this range, for
+  /// example during an overscroll.
+  ///
   /// The `rect` is the region occupied by this node in its own coordinate
   /// system.
   ///
@@ -348,6 +355,9 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
     int actions,
     int textSelectionBase,
     int textSelectionExtent,
+    double scrollPosition,
+    double scrollExtentMax,
+    double scrollExtentMin,
     Rect rect,
     String label,
     String hint,
@@ -366,6 +376,9 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
                 actions,
                 textSelectionBase,
                 textSelectionExtent,
+                scrollPosition,
+                scrollExtentMax,
+                scrollExtentMin,
                 rect.left,
                 rect.top,
                 rect.right,
@@ -386,6 +399,9 @@ class SemanticsUpdateBuilder extends NativeFieldWrapperClass2 {
     int actions,
     int textSelectionBase,
     int textSelectionExtent,
+    double scrollPosition,
+    double scrollExtentMax,
+    double scrollExtentMin,
     double left,
     double top,
     double right,

lib/ui/semantics/semantics_node.h

@@ -58,6 +58,9 @@ struct SemanticsNode {
   int32_t actions = 0;
   int32_t textSelectionBase = -1;
   int32_t textSelectionExtent = -1;
+  double scrollPosition = std::nan("");
+  double scrollExtentMax = std::nan("");
+  double scrollExtentMin = std::nan("");
   std::string label;
   std::string hint;
   std::string value;

lib/ui/semantics/semantics_update_builder.cc

@@ -39,6 +39,9 @@ void SemanticsUpdateBuilder::updateNode(int id,
                                         int actions,
                                         int textSelectionBase,
                                         int textSelectionExtent,
+                                        double scrollPosition,
+                                        double scrollExtentMax,
+                                        double scrollExtentMin,
                                         double left,
                                         double top,
                                         double right,
@@ -58,6 +61,9 @@ void SemanticsUpdateBuilder::updateNode(int id,
   node.actions = actions;
   node.textSelectionBase = textSelectionBase;
   node.textSelectionExtent = textSelectionExtent;
+  node.scrollPosition = scrollPosition;
+  node.scrollExtentMax = scrollExtentMax;
+  node.scrollExtentMin = scrollExtentMin;
   node.rect = SkRect::MakeLTRB(left, top, right, bottom);
   node.label = label;
   node.hint = hint;

lib/ui/semantics/semantics_update_builder.h

@@ -30,6 +30,9 @@ class SemanticsUpdateBuilder
                   int actions,
                   int textSelectionBase,
                   int textSelectionExtent,
+                  double scrollPosition,
+                  double scrollExtentMax,
+                  double scrollExtentMin,
                   double left,
                   double top,
                   double right,

shell/platform/android/io/flutter/view/AccessibilityBridge.java

@@ -34,7 +34,10 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
     // Constants from higher API levels.
     // TODO(goderbauer): Get these from Android Support Library when
     // https://github.com/flutter/flutter/issues/11099 is resolved.
-    public static final int ACTION_SHOW_ON_SCREEN = 16908342; // API level 23
+    private static final int ACTION_SHOW_ON_SCREEN = 16908342; // API level 23
+
+    private static final float SCROLL_EXTENT_FOR_INFINITY = 100000.0f;
+    private static final float SCROLL_POSITION_CAP_FOR_INFINITY = 70000.0f;
 
     private Map<Integer, SemanticsObject> mObjects;
     private final FlutterView mOwner;
@@ -460,7 +463,9 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
             if (object.hasFlag(Flag.IS_FOCUSED)) {
                 mInputFocusedObject = object;
             }
-            updated.add(object);
+            if (object.hadPreviousConfig) {
+                updated.add(object);
+            }
         }
 
         Set<SemanticsObject> visitedObjects = new HashSet<SemanticsObject>();
@@ -481,13 +486,46 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
             }
         }
 
-        // Send accessibility events for updated nodes
+        // TODO(goderbauer): Send this event only once (!) for changed subtrees,
+        //     see https://github.com/flutter/flutter/issues/14534
+        sendAccessibilityEvent(0, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
+
         for (SemanticsObject object : updated) {
-            sendAccessibilityEvent(object.id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
-            if (!object.hadPreviousConfig) {
-                continue;
-            }
+            if (object.didScroll()) {
+                AccessibilityEvent event =
+                        obtainAccessibilityEvent(object.id, AccessibilityEvent.TYPE_VIEW_SCROLLED);
+
+                // Android doesn't support unbound scrolling. So we pretend there is a large
+                // bound (SCROLL_EXTENT_FOR_INFINITY), which you can never reach.
+                float position = object.scrollPosition;
+                float max = object.scrollExtentMax;
+                if (Float.isInfinite(object.scrollExtentMax)) {
+                    max = SCROLL_EXTENT_FOR_INFINITY;
+                    if (position > SCROLL_POSITION_CAP_FOR_INFINITY) {
+                        position = SCROLL_POSITION_CAP_FOR_INFINITY;
+                    }
+                }
+                if (Float.isInfinite(object.scrollExtentMin)) {
+                    max += SCROLL_EXTENT_FOR_INFINITY;
+                    if (position < -SCROLL_POSITION_CAP_FOR_INFINITY) {
+                        position = -SCROLL_POSITION_CAP_FOR_INFINITY;
+                    }
+                    position += SCROLL_EXTENT_FOR_INFINITY;
+                } else {
+                    max -= object.scrollExtentMin;
+                    position -= object.scrollExtentMin;
+                }
 
+                if (object.hadAction(Action.SCROLL_UP) || object.hadAction(Action.SCROLL_DOWN)) {
+                    event.setScrollY((int) position);
+                    event.setMaxScrollY((int) max);
+                } else if (object.hadAction(Action.SCROLL_LEFT)
+                        || object.hadAction(Action.SCROLL_RIGHT)) {
+                    event.setScrollX((int) position);
+                    event.setMaxScrollX((int) max);
+                }
+                sendAccessibilityEvent(event);
+            }
             if (mA11yFocusedObject != null && mA11yFocusedObject.id == object.id
                     && object.hadFlag(Flag.HAS_CHECKED_STATE)
                     && object.hasFlag(Flag.HAS_CHECKED_STATE)
@@ -586,24 +624,6 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         final HashMap<String, Object> data = (HashMap<String, Object>)annotatedEvent.get("data");
 
         switch (type) {
-            case "scroll":
-                final int nodeId = (int)annotatedEvent.get("nodeId");
-                AccessibilityEvent event =
-                    obtainAccessibilityEvent(nodeId, AccessibilityEvent.TYPE_VIEW_SCROLLED);
-                char axis = ((String)data.get("axis")).charAt(0);
-                double minPosition = (double)data.get("minScrollExtent");
-                double maxPosition = (double)data.get("maxScrollExtent") - minPosition;
-                double position = (double)data.get("pixels") - minPosition;
-                if (axis == 'v') {
-                    event.setScrollY((int)position);
-                    event.setMaxScrollY((int)maxPosition);
-                } else {
-                    assert axis == 'h';
-                    event.setScrollX((int)position);
-                    event.setMaxScrollX((int)maxPosition);
-                }
-                sendAccessibilityEvent(event);
-                break;
             case "announce":
                 mOwner.announceForAccessibility((String) data.get("message"));
                 break;
@@ -660,6 +680,9 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
         int actions;
         int textSelectionBase;
         int textSelectionExtent;
+        float scrollPosition;
+        float scrollExtentMax;
+        float scrollExtentMin;
         String label;
         String value;
         String increasedValue;
@@ -670,8 +693,12 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
 
         boolean hadPreviousConfig = false;
         int previousFlags;
+        int previousActions;
         int previousTextSelectionBase;
         int previousTextSelectionExtent;
+        float previousScrollPosition;
+        float previousScrollExtentMax;
+        float previousScrollExtentMin;
         String previousValue;
 
         private float left;
@@ -694,6 +721,10 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
             return (actions & action.value) != 0;
         }
 
+        boolean hadAction(Action action) {
+            return (previousActions & action.value) != 0;
+        }
+
         boolean hasFlag(Flag flag) {
             return (flags & flag.value) != 0;
         }
@@ -703,6 +734,11 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
             return (previousFlags & flag.value) != 0;
         }
 
+        boolean didScroll() {
+            return !Float.isNaN(scrollPosition) && !Float.isNaN(previousScrollPosition)
+                    && previousScrollPosition != scrollPosition;
+        }
+
         void log(String indent, boolean recursive) {
           Log.i(TAG, indent + "SemanticsObject id=" + id + " label=" + label + " actions=" +  actions + " flags=" + flags + "\n" +
                      indent + "  +-- textDirection=" + textDirection  + "\n"+
@@ -721,13 +757,20 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
             hadPreviousConfig = true;
             previousValue = value;
             previousFlags = flags;
+            previousActions = actions;
             previousTextSelectionBase = textSelectionBase;
             previousTextSelectionExtent = textSelectionExtent;
+            previousScrollPosition = scrollPosition;
+            previousScrollExtentMax = scrollExtentMax;
+            previousScrollExtentMin = scrollExtentMin;
 
             flags = buffer.getInt();
             actions = buffer.getInt();
             textSelectionBase = buffer.getInt();
             textSelectionExtent = buffer.getInt();
+            scrollPosition = buffer.getFloat();
+            scrollExtentMax = buffer.getFloat();
+            scrollExtentMin = buffer.getFloat();
 
             int stringIndex = buffer.getInt();
             label = stringIndex == -1 ? null : strings[stringIndex];

shell/platform/android/platform_view_android.cc

@@ -461,7 +461,7 @@ bool PlatformViewAndroid::ResourceContextMakeCurrent() {
 
 void PlatformViewAndroid::UpdateSemantics(
     blink::SemanticsNodeUpdates update) {
-  constexpr size_t kBytesPerNode = 33 * sizeof(int32_t);
+  constexpr size_t kBytesPerNode = 36 * sizeof(int32_t);
   constexpr size_t kBytesPerChild = sizeof(int32_t);
 
   JNIEnv* env = fml::jni::AttachCurrentThread();
@@ -492,6 +492,9 @@ void PlatformViewAndroid::UpdateSemantics(
       buffer_int32[position++] = node.actions;
       buffer_int32[position++] = node.textSelectionBase;
       buffer_int32[position++] = node.textSelectionExtent;
+      buffer_float32[position++] = (float)node.scrollPosition;
+      buffer_float32[position++] = (float)node.scrollExtentMax;
+      buffer_float32[position++] = (float)node.scrollExtentMin;
       if (node.label.empty()) {
         buffer_int32[position++] = -1;
       } else {

shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm

@@ -143,6 +143,14 @@ bool GeometryComparator(SemanticsObject* a, SemanticsObject* b) {
   return [self node].rect != node->rect || [self node].transform != node->transform;
 }
 
+/**
+ * Whether calling `setSemanticsNode:` with `node` would cause a scroll event.
+ */
+- (BOOL)nodeWillCauseScroll:(const blink::SemanticsNode*)node {
+  return !isnan([self node].scrollPosition) && !isnan(node->scrollPosition) &&
+         [self node].scrollPosition != node->scrollPosition;
+}
+
 - (std::vector<SemanticsObject*>*)children {
   return &_children;
 }
@@ -404,11 +412,13 @@ void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes) {
   // traversal order (top left to bottom right, with hit testing order as tie breaker).
   NSMutableSet<SemanticsObject*>* childOrdersToUpdate = [[[NSMutableSet alloc] init] autorelease];
   BOOL layoutChanged = NO;
+  BOOL scrollOccured = NO;
 
   for (const auto& entry : nodes) {
     const blink::SemanticsNode& node = entry.second;
     SemanticsObject* object = GetOrCreateObject(node.id, nodes);
     layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
+    scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
     [object setSemanticsNode:&node];
     const size_t childrenCount = node.children.size();
     auto& children = *[object children];
@@ -452,6 +462,10 @@ void AccessibilityBridge::UpdateSemantics(blink::SemanticsNodeUpdates nodes) {
     // TODO(goderbauer): figure out which node to focus next.
     UIAccessibilityPostNotification(UIAccessibilityLayoutChangedNotification, nil);
   }
+  if (scrollOccured) {
+    // TODO(tvolkert): provide meaningful string (e.g. "page 2 of 5")
+    UIAccessibilityPostNotification(UIAccessibilityPageScrolledNotification, @"");
+  }
 }
 
 void AccessibilityBridge::DispatchSemanticsAction(int32_t uid, blink::SemanticsAction action) {
@@ -511,10 +525,7 @@ void AccessibilityBridge::VisitObjectsRecursivelyAndRemove(SemanticsObject* obje
 
 void AccessibilityBridge::HandleEvent(NSDictionary<NSString*, id>* annotatedEvent) {
   NSString* type = annotatedEvent[@"type"];
-  if ([type isEqualToString:@"scroll"]) {
-    // TODO(tvolkert): provide meaningful string (e.g. "page 2 of 5")
-    UIAccessibilityPostNotification(UIAccessibilityPageScrolledNotification, @"");
-  } else if ([type isEqualToString:@"announce"]) {
+  if ([type isEqualToString:@"announce"]) {
     NSString* message = annotatedEvent[@"data"][@"message"];
     UIAccessibilityPostNotification(UIAccessibilityAnnouncementNotification, message);
   } else {