Improve new semantics backend (#3113)

TalkBack still isn't fully working, but this patch gets us much closer.

runtime/runtime_controller.cc

@@ -29,19 +29,24 @@ void RuntimeController::CreateDartController(const std::string& script_uri) {
   FTL_DCHECK(!dart_controller_);
 
   dart_controller_.reset(new DartController());
-  std::unique_ptr<Window> window(new Window(this));
   dart_controller_->CreateIsolateFor(
       script_uri,
-      std::unique_ptr<UIDartState>(new UIDartState(this, std::move(window))));
+      std::make_unique<UIDartState>(this, std::make_unique<Window>(this)));
 
   UIDartState* dart_state = dart_controller_->dart_state();
   DartState::Scope scope(dart_state);
   dart_state->window()->DidCreateIsolate();
   client_->DidCreateMainIsolate(dart_state->isolate());
 
+  Window* window = GetWindow();
+
   if (viewport_metrics_)
-    GetWindow()->UpdateWindowMetrics(viewport_metrics_);
-  GetWindow()->UpdateLocale(language_code_, country_code_);
+    window->UpdateWindowMetrics(viewport_metrics_);
+
+  window->UpdateLocale(language_code_, country_code_);
+
+  if (semantics_enabled_)
+    window->UpdateSemanticsEnabled(semantics_enabled_);
 }
 
 void RuntimeController::SetViewportMetrics(
@@ -68,7 +73,7 @@ void RuntimeController::SetSemanticsEnabled(bool enabled) {
   if (semantics_enabled_ == enabled)
     return;
   semantics_enabled_ = enabled;
-  GetWindow()->UpdateSemanticsEnabled(enabled);
+  GetWindow()->UpdateSemanticsEnabled(semantics_enabled_);
 }
 
 void RuntimeController::PushRoute(const std::string& route) {

shell/common/engine.cc

@@ -172,8 +172,9 @@ void Engine::DispatchSemanticsAction(int id, blink::SemanticsAction action) {
 
 void Engine::SetSemanticsEnabled(bool enabled) {
   TRACE_EVENT0("flutter", "Engine::DispatchPointerDataPacket");
+  semantics_enabled_ = enabled;
   if (runtime_)
-    runtime_->SetSemanticsEnabled(enabled);
+    runtime_->SetSemanticsEnabled(semantics_enabled_);
 }
 
 void Engine::RunFromSnapshotStream(
@@ -220,6 +221,7 @@ void Engine::ConfigureRuntime(const std::string& script_uri) {
   runtime_->CreateDartController(std::move(script_uri));
   runtime_->SetViewportMetrics(viewport_metrics_);
   runtime_->SetLocale(language_code_, country_code_);
+  runtime_->SetSemanticsEnabled(semantics_enabled_);
   if (!initial_route_.empty())
     runtime_->PushRoute(initial_route_);
 }

shell/common/engine.h

@@ -111,6 +111,7 @@ class Engine : public sky::SkyEngine, public blink::RuntimeDelegate {
   sky::ViewportMetricsPtr viewport_metrics_;
   std::string language_code_;
   std::string country_code_;
+  bool semantics_enabled_ = false;
   mojo::Binding<SkyEngine> binding_;
   ftl::RefPtr<blink::ZipAssetStore> asset_store_;
 

shell/platform/android/io/flutter/view/AccessibilityBridge.java

@@ -245,7 +245,6 @@ class AccessibilityBridge extends AccessibilityNodeProvider {
     }
 
     void updateSemantics(ByteBuffer buffer, String[] strings) {
-        buffer.order(ByteOrder.LITTLE_ENDIAN);
         ArrayList<Integer> updated = new ArrayList<Integer>();
         while (buffer.hasRemaining()) {
             int id = buffer.getInt();
@@ -264,8 +263,9 @@ class AccessibilityBridge extends AccessibilityNodeProvider {
         Iterator<Map.Entry<Integer, SemanticsObject>> it = mObjects.entrySet().iterator();
         while (it.hasNext()) {
             Map.Entry<Integer, SemanticsObject> entry = it.next();
-            if (!visitedObjects.contains(entry.getKey())) {
-                willRemoveSemanticsObject(entry.getValue());
+            SemanticsObject object = entry.getValue();
+            if (!visitedObjects.contains(object)) {
+                willRemoveSemanticsObject(object);
                 it.remove();
             }
         }
@@ -316,8 +316,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider {
 
         int id = -1;
 
-        int actions;
         int flags;
+        int actions;
         String label;
 
         private float left;
@@ -336,9 +336,21 @@ class AccessibilityBridge extends AccessibilityNodeProvider {
         private float[] globalTransform;
         private Rect globalRect;
 
+        void log(String indent) {
+          Log.i(TAG, indent + "SemanticsObject id=" + id + " label=" + label + " actions=" +  actions + " flags=" + flags + "\n" +
+                     indent + "  +-- rect.ltrb=(" + left + ", " + top + ", " + right + ", " + bottom + ")\n" +
+                     indent + "  +-- transform=" + Arrays.toString(transform) + "\n");
+          if (children != null) {
+              String childIndent = indent + "  ";
+              for (SemanticsObject child : children) {
+                  child.log(childIndent);
+              }
+          }
+        }
+
         void updateWith(ByteBuffer buffer, String[] strings) {
-            actions = buffer.getInt();
             flags = buffer.getInt();
+            actions = buffer.getInt();
 
             final int stringIndex = buffer.getInt();
             if (stringIndex == -1)
@@ -369,8 +381,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider {
 
                 for (int i = 0; i < childCount; ++i) {
                     SemanticsObject child = getOrCreateObject(buffer.getInt());
-                    children.add(child);
                     child.parent = this;
+                    children.add(child);
                 }
             }
         }

shell/platform/android/io/flutter/view/FlutterView.java

@@ -608,8 +608,10 @@ public class FlutterView extends SurfaceView
 
     @CalledByNative
     private void updateSemantics(ByteBuffer buffer, String[] strings) {
-        if (mAccessibilityNodeProvider != null)
+        if (mAccessibilityNodeProvider != null) {
+            buffer.order(ByteOrder.LITTLE_ENDIAN);
             mAccessibilityNodeProvider.updateSemantics(buffer, strings);
+        }
     }
 
     // ACCESSIBILITY