From 2dc88cc6a5e4192ba10669ebed4b034a71b9b158 Mon Sep 17 00:00:00 2001 From: Adam Barth Date: Tue, 11 Oct 2016 10:52:48 -0700 Subject: [PATCH] Switch backend to consume new semantics API (#3103) --- common/threads.cc | 4 + lib/ui/semantics/semantics_node.cc | 8 + lib/ui/semantics/semantics_node.h | 3 + shell/common/animator.cc | 31 +- shell/common/animator.h | 2 +- shell/common/engine.cc | 28 +- shell/common/engine.h | 6 +- shell/common/platform_view.cc | 32 +- shell/common/platform_view.h | 20 +- .../io/flutter/view/AccessibilityBridge.java | 480 +++++++++--------- .../android/io/flutter/view/FlutterView.java | 27 +- .../platform/android/platform_view_android.cc | 73 ++- .../platform/android/platform_view_android.h | 19 +- .../darwin/desktop/platform_view_mac.h | 3 - .../darwin/desktop/platform_view_mac.mm | 7 +- shell/platform/darwin/ios/BUILD.gn | 1 + .../framework/Source/FlutterViewController.mm | 6 +- .../framework/Source/accessibility_bridge.h | 46 +- .../framework/Source/accessibility_bridge.mm | 294 ++++------- shell/platform/darwin/ios/platform_view_ios.h | 7 +- .../platform/darwin/ios/platform_view_ios.mm | 22 +- shell/platform/linux/platform_view_glfw.cc | 7 +- shell/platform/linux/platform_view_glfw.h | 3 - shell/testing/platform_view_test.cc | 9 +- shell/testing/platform_view_test.h | 4 - 25 files changed, 576 insertions(+), 566 deletions(-) diff --git a/common/threads.cc b/common/threads.cc index 8fc15125b..df2265374 100644 --- a/common/threads.cc +++ b/common/threads.cc @@ -26,6 +26,10 @@ Threads::Threads(ftl::RefPtr platform, Threads::~Threads() {} +const ftl::RefPtr& Threads::Platform() { + return Get().platform_; +} + const ftl::RefPtr& Threads::Gpu() { return Get().gpu_; } diff --git a/lib/ui/semantics/semantics_node.cc b/lib/ui/semantics/semantics_node.cc index 4d30f0920..563ca1d89 100644 --- a/lib/ui/semantics/semantics_node.cc +++ b/lib/ui/semantics/semantics_node.cc @@ -12,4 +12,12 @@ SemanticsNode::SemanticsNode() = default; SemanticsNode::~SemanticsNode() = default; +bool SemanticsNode::HasAction(SemanticsAction action) { + return (actions & static_cast(action)) != 0; +} + +bool SemanticsNode::HasFlag(SemanticsFlags flag) { + return (flags & static_cast(flag)) != 0; +} + } // namespace blink diff --git a/lib/ui/semantics/semantics_node.h b/lib/ui/semantics/semantics_node.h index 47474baf2..e221d3173 100644 --- a/lib/ui/semantics/semantics_node.h +++ b/lib/ui/semantics/semantics_node.h @@ -37,6 +37,9 @@ struct SemanticsNode { SemanticsNode(); ~SemanticsNode(); + bool HasAction(SemanticsAction action); + bool HasFlag(SemanticsFlags flag); + int32_t id = 0; int32_t flags = 0; int32_t actions = 0; diff --git a/shell/common/animator.cc b/shell/common/animator.cc index d7f419ed1..ddedfc8eb 100644 --- a/shell/common/animator.cc +++ b/shell/common/animator.cc @@ -12,8 +12,8 @@ namespace shell { -Animator::Animator(Rasterizer* rasterizer, Engine* engine) - : rasterizer_(rasterizer->GetWeakRasterizerPtr()), +Animator::Animator(ftl::WeakPtr rasterizer, Engine* engine) + : rasterizer_(rasterizer), engine_(engine), layer_tree_pipeline_(ftl::MakeRefCounted(3)), pending_frame_semaphore_(1), @@ -77,16 +77,12 @@ void Animator::Render(std::unique_ptr layer_tree) { // Commit the pending continuation. producer_continuation_.Complete(std::move(layer_tree)); - // Notify the rasterizer that the pipeline has items it may consume. - auto weak_rasterizer(rasterizer_); - auto pipeline = layer_tree_pipeline_; - - blink::Threads::Gpu()->PostTask([weak_rasterizer, pipeline]() { - if (!weak_rasterizer) { - return; - } - weak_rasterizer->Draw(pipeline); - }); + blink::Threads::Gpu()->PostTask( + [ rasterizer = rasterizer_, pipeline = layer_tree_pipeline_ ]() { + if (!rasterizer.get()) + return; + rasterizer->Draw(pipeline); + }); } void Animator::RequestFrame() { @@ -107,16 +103,11 @@ void Animator::RequestFrame() { // started an expensive operation right after posting this message however. // To support that, we need edge triggered wakes on VSync. - auto weak = weak_factory_.GetWeakPtr(); - - blink::Threads::UI()->PostTask([weak]() { - if (!weak) { + blink::Threads::UI()->PostTask([self = weak_factory_.GetWeakPtr()]() { + if (!self.get()) return; - } - TRACE_EVENT_INSTANT0("flutter", "RequestFrame", TRACE_EVENT_SCOPE_PROCESS); - - weak->AwaitVSync(base::Bind(&Animator::BeginFrame, weak)); + self->AwaitVSync(base::Bind(&Animator::BeginFrame, self)); }); } diff --git a/shell/common/animator.h b/shell/common/animator.h index 2f95c5927..107cfa511 100644 --- a/shell/common/animator.h +++ b/shell/common/animator.h @@ -19,7 +19,7 @@ namespace shell { class Animator { public: - explicit Animator(Rasterizer* rasterizer, Engine* engine); + explicit Animator(ftl::WeakPtr rasterizer, Engine* engine); ~Animator(); diff --git a/shell/common/engine.cc b/shell/common/engine.cc index e3f124fd1..53e3b18bc 100644 --- a/shell/common/engine.cc +++ b/shell/common/engine.cc @@ -20,8 +20,10 @@ #include "flutter/runtime/dart_init.h" #include "flutter/runtime/runtime_init.h" #include "flutter/shell/common/animator.h" +#include "flutter/shell/common/platform_view.h" #include "flutter/sky/engine/public/web/Sky.h" #include "lib/ftl/files/path.h" +#include "lib/ftl/functional/make_copyable.h" #include "mojo/public/cpp/application/connect.h" #include "third_party/skia/include/core/SkCanvas.h" #include "third_party/skia/include/core/SkPictureRecorder.h" @@ -47,8 +49,10 @@ std::string FindPackagesPath(const std::string& main_dart) { } // namespace -Engine::Engine(Rasterizer* rasterizer) - : animator_(new Animator(rasterizer, this)), +Engine::Engine(PlatformView* platform_view) + : platform_view_(platform_view->GetWeakPtr()), + animator_(new Animator(platform_view->rasterizer().GetWeakRasterizerPtr(), + this)), binding_(this), activity_running_(false), have_surface_(false), @@ -160,6 +164,18 @@ void Engine::DispatchPointerDataPacket(const PointerDataPacket& packet) { runtime_->DispatchPointerDataPacket(packet); } +void Engine::DispatchSemanticsAction(int id, blink::SemanticsAction action) { + TRACE_EVENT0("flutter", "Engine::DispatchPointerDataPacket"); + if (runtime_) + runtime_->DispatchSemanticsAction(id, action); +} + +void Engine::SetSemanticsEnabled(bool enabled) { + TRACE_EVENT0("flutter", "Engine::DispatchPointerDataPacket"); + if (runtime_) + runtime_->SetSemanticsEnabled(enabled); +} + void Engine::RunFromSnapshotStream( const std::string& script_uri, mojo::ScopedDataPipeConsumerHandle snapshot) { @@ -334,6 +350,12 @@ void Engine::Render(std::unique_ptr layer_tree) { animator_->Render(std::move(layer_tree)); } -void Engine::UpdateSemantics(std::vector update) {} +void Engine::UpdateSemantics(std::vector update) { + blink::Threads::Platform()->PostTask(ftl::MakeCopyable( + [ platform_view = platform_view_, update = std::move(update) ]() mutable { + if (platform_view) + platform_view->UpdateSemantics(std::move(update)); + })); +} } // namespace shell diff --git a/shell/common/engine.h b/shell/common/engine.h index ce7a3a33d..6e21db9fd 100644 --- a/shell/common/engine.h +++ b/shell/common/engine.h @@ -24,12 +24,13 @@ #include "third_party/skia/include/core/SkPicture.h" namespace shell { +class PlatformView; class Animator; using PointerDataPacket = blink::PointerDataPacket; class Engine : public sky::SkyEngine, public blink::RuntimeDelegate { public: - explicit Engine(Rasterizer* rasterizer); + explicit Engine(PlatformView* platform_view); ~Engine() override; @@ -51,6 +52,8 @@ class Engine : public sky::SkyEngine, public blink::RuntimeDelegate { void OnOutputSurfaceCreated(const ftl::Closure& gpu_continuation); void OnOutputSurfaceDestroyed(const ftl::Closure& gpu_continuation); void DispatchPointerDataPacket(const PointerDataPacket& packet); + void DispatchSemanticsAction(int id, blink::SemanticsAction action); + void SetSemanticsEnabled(bool enabled); private: // SkyEngine implementation: @@ -91,6 +94,7 @@ class Engine : public sky::SkyEngine, public blink::RuntimeDelegate { void ConfigureAssetBundle(const std::string& path); void ConfigureRuntime(const std::string& script_uri); + ftl::WeakPtr platform_view_; std::unique_ptr animator_; sky::ServicesDataPtr services_; diff --git a/shell/common/platform_view.cc b/shell/common/platform_view.cc index 9a39b0d8f..152af5756 100644 --- a/shell/common/platform_view.cc +++ b/shell/common/platform_view.cc @@ -15,8 +15,10 @@ namespace shell { PlatformView::PlatformView(std::unique_ptr rasterizer) - : rasterizer_(std::move(rasterizer)), size_(SkISize::Make(0, 0)) { - engine_.reset(new Engine(rasterizer_.get())); + : rasterizer_(std::move(rasterizer)), + size_(SkISize::Make(0, 0)), + weak_factory_(this) { + engine_.reset(new Engine(this)); } PlatformView::~PlatformView() { @@ -30,10 +32,28 @@ PlatformView::~PlatformView() { blink::Threads::UI()->PostTask([engine]() { delete engine; }); } +void PlatformView::DispatchSemanticsAction(int32_t id, + blink::SemanticsAction action) { + blink::Threads::UI()->PostTask( + [ engine = engine_->GetWeakPtr(), id, action ] { + if (engine.get()) { + engine->DispatchSemanticsAction( + id, static_cast(action)); + } + }); +} + +void PlatformView::SetSemanticsEnabled(bool enabled) { + blink::Threads::UI()->PostTask([ engine = engine_->GetWeakPtr(), enabled ] { + if (engine.get()) + engine->SetSemanticsEnabled(enabled); + }); +} + void PlatformView::ConnectToEngine( mojo::InterfaceRequest request) { blink::Threads::UI()->PostTask(ftl::MakeCopyable([ - view = GetWeakViewPtr(), engine = engine().GetWeakPtr(), + view = GetWeakPtr(), engine = engine().GetWeakPtr(), request = std::move(request) ]() mutable { if (engine.get()) @@ -90,6 +110,10 @@ void PlatformView::NotifyDestroyed() { latch.Wait(); } +ftl::WeakPtr PlatformView::GetWeakPtr() { + return weak_factory_.GetWeakPtr(); +} + SkISize PlatformView::GetSize() { return size_; } @@ -98,6 +122,8 @@ void PlatformView::Resize(const SkISize& size) { size_ = size; } +void PlatformView::UpdateSemantics(std::vector update) {} + void PlatformView::SetupResourceContextOnIOThread() { ftl::AutoResetWaitableEvent latch; diff --git a/shell/common/platform_view.h b/shell/common/platform_view.h index 7e2038145..3aa28797a 100644 --- a/shell/common/platform_view.h +++ b/shell/common/platform_view.h @@ -7,6 +7,7 @@ #include +#include "flutter/lib/ui/semantics/semantics_node.h" #include "flutter/shell/common/engine.h" #include "flutter/shell/common/shell.h" #include "flutter/shell/common/surface.h" @@ -35,6 +36,9 @@ class PlatformView { virtual ~PlatformView(); + void DispatchSemanticsAction(int32_t id, blink::SemanticsAction action); + void SetSemanticsEnabled(bool enabled); + void ConnectToEngine(mojo::InterfaceRequest request); void NotifyCreated(std::unique_ptr surface); @@ -44,7 +48,7 @@ class PlatformView { void NotifyDestroyed(); - virtual ftl::WeakPtr GetWeakViewPtr() = 0; + ftl::WeakPtr GetWeakPtr(); virtual bool ResourceContextMakeCurrent() = 0; @@ -52,6 +56,8 @@ class PlatformView { virtual void Resize(const SkISize& size); + virtual void UpdateSemantics(std::vector update); + Rasterizer& rasterizer() { return *rasterizer_; } Engine& engine() { return *engine_; } @@ -60,17 +66,19 @@ class PlatformView { const std::string& assets_directory) = 0; protected: - SurfaceConfig surface_config_; - std::unique_ptr rasterizer_; - std::unique_ptr engine_; - SkISize size_; - explicit PlatformView(std::unique_ptr rasterizer); void SetupResourceContextOnIOThreadPerform( ftl::AutoResetWaitableEvent* event); + SurfaceConfig surface_config_; + std::unique_ptr rasterizer_; + std::unique_ptr engine_; + SkISize size_; + private: + ftl::WeakPtrFactory weak_factory_; + FTL_DISALLOW_COPY_AND_ASSIGN(PlatformView); }; diff --git a/shell/platform/android/io/flutter/view/AccessibilityBridge.java b/shell/platform/android/io/flutter/view/AccessibilityBridge.java index 73b08ccb5..dc87818e3 100644 --- a/shell/platform/android/io/flutter/view/AccessibilityBridge.java +++ b/shell/platform/android/io/flutter/view/AccessibilityBridge.java @@ -7,41 +7,54 @@ package io.flutter.view; import android.graphics.Rect; import android.opengl.Matrix; import android.os.Bundle; +import android.util.Log; import android.view.View; import android.view.accessibility.AccessibilityEvent; import android.view.accessibility.AccessibilityManager; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityNodeProvider; -import org.chromium.mojo.system.MojoException; -import org.chromium.mojom.semantics.SemanticAction; -import org.chromium.mojom.semantics.SemanticsListener; -import org.chromium.mojom.semantics.SemanticsNode; -import org.chromium.mojom.semantics.SemanticsServer; -import org.chromium.mojom.sky.ViewportMetrics; - +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -class AccessibilityBridge extends AccessibilityNodeProvider implements SemanticsListener { - private Map mObjects; +class AccessibilityBridge extends AccessibilityNodeProvider { + private static final String TAG = "FlutterView"; + + private Map mObjects; private FlutterView mOwner; - private SemanticsServer.Proxy mSemanticsServer; private boolean mAccessibilityEnabled = false; - private SemanticObject mFocusedObject; - private SemanticObject mHoveredObject; - - AccessibilityBridge(FlutterView owner, SemanticsServer.Proxy semanticsServer) { + private SemanticsObject mFocusedObject; + private SemanticsObject mHoveredObject; + + private static final int SEMANTICS_ACTION_TAP = 1 << 0; + private static final int SEMANTICS_ACTION_LONG_PRESS = 1 << 1; + private static final int SEMANTICS_ACTION_SCROLL_LEFT = 1 << 2; + private static final int SEMANTICS_ACTION_SCROLL_RIGHT = 1 << 3; + private static final int SEMANTICS_ACTION_SCROLL_UP = 1 << 4; + private static final int SEMANTICS_ACTION_SCROLL_DOWN = 1 << 5; + private static final int SEMANTICS_ACTION_INCREASE = 1 << 6; + private static final int SEMANTICS_ACTION_DECREASE = 1 << 7; + + private static final int SEMANTICS_ACTION_SCROLLABLE = SEMANTICS_ACTION_SCROLL_LEFT | + SEMANTICS_ACTION_SCROLL_RIGHT | + SEMANTICS_ACTION_SCROLL_UP | + SEMANTICS_ACTION_SCROLL_DOWN; + + private static final int SEMANTICS_FLAG_HAS_CHECKED_STATE = 1 << 0; + private static final int SEMANTICS_FLAG_IS_CHECKED = 1 << 1; + + AccessibilityBridge(FlutterView owner) { assert owner != null; - assert semanticsServer != null; mOwner = owner; - mObjects = new HashMap(); - mSemanticsServer = semanticsServer; - mSemanticsServer.addSemanticsListener(this); + mObjects = new HashMap(); } void setAccessibilityEnabled(boolean accessibilityEnabled) { @@ -58,7 +71,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics return result; } - SemanticObject object = mObjects.get(virtualViewId); + SemanticsObject object = mObjects.get(virtualViewId); if (object == null) return null; @@ -88,15 +101,15 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics result.setVisibleToUser(true); result.setEnabled(true); // TODO(ianh): Expose disabled subtrees - if (object.canBeTapped) { + if ((object.actions & SEMANTICS_ACTION_TAP) != 0) { result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_CLICK); result.setClickable(true); } - if (object.canBeLongPressed) { + if ((object.actions & SEMANTICS_ACTION_LONG_PRESS) != 0) { result.addAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_LONG_CLICK); result.setLongClickable(true); } - if (object.canBeScrolledHorizontally || object.canBeScrolledVertically) { + if ((object.actions & SEMANTICS_ACTION_SCROLLABLE) != 0) { // TODO(ianh): Once we're on SDK v23+, call addAction to // expose AccessibilityAction.ACTION_SCROLL_LEFT, _RIGHT, // _UP, and _DOWN when appropriate. @@ -106,8 +119,8 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics result.setScrollable(true); } - result.setCheckable(object.hasCheckedState); - result.setChecked(object.isChecked); + result.setCheckable((object.flags & SEMANTICS_FLAG_HAS_CHECKED_STATE) != 0); + result.setChecked((object.flags & SEMANTICS_FLAG_IS_CHECKED) != 0); result.setText(object.label); // TODO(ianh): use setTraversalBefore/setTraversalAfter to set @@ -124,7 +137,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics } if (object.children != null) { - for (SemanticObject child : object.children) { + for (SemanticsObject child : object.children) { result.addChild(mOwner, child.id); } } @@ -134,36 +147,36 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics @Override public boolean performAction(int virtualViewId, int action, Bundle arguments) { - SemanticObject object = mObjects.get(virtualViewId); + SemanticsObject object = mObjects.get(virtualViewId); if (object == null) { return false; } switch (action) { case AccessibilityNodeInfo.ACTION_CLICK: { - mSemanticsServer.performAction(virtualViewId, SemanticAction.TAP); + mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_TAP); return true; } case AccessibilityNodeInfo.ACTION_LONG_CLICK: { - mSemanticsServer.performAction(virtualViewId, SemanticAction.LONG_PRESS); + mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_LONG_PRESS); return true; } case AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD: { - if (object.canBeScrolledVertically) { - mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_UP); - } else if (object.canBeScrolledHorizontally) { + if ((object.actions & SEMANTICS_ACTION_SCROLL_UP) != 0) { + mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_SCROLL_UP); + } else if ((object.actions & SEMANTICS_ACTION_SCROLL_LEFT) != 0) { // TODO(ianh): bidi support - mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_LEFT); + mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_SCROLL_LEFT); } else { return false; } return true; } case AccessibilityNodeInfo.ACTION_SCROLL_FORWARD: { - if (object.canBeScrolledVertically) { - mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_DOWN); - } else if (object.canBeScrolledHorizontally) { + if ((object.actions & SEMANTICS_ACTION_SCROLL_DOWN) != 0) { + mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_SCROLL_DOWN); + } else if ((object.actions & SEMANTICS_ACTION_SCROLL_RIGHT) != 0) { // TODO(ianh): bidi support - mSemanticsServer.performAction(virtualViewId, SemanticAction.SCROLL_RIGHT); + mOwner.dispatchSemanticsAction(virtualViewId, SEMANTICS_ACTION_SCROLL_RIGHT); } else { return false; } @@ -192,10 +205,21 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics // TODO(ianh): implement findAccessibilityNodeInfosByText() // TODO(ianh): implement findFocus() - private SemanticObject getRootObject() { + private SemanticsObject getRootObject() { + assert mObjects.containsKey(0); return mObjects.get(0); } + private SemanticsObject getOrCreateObject(int id) { + SemanticsObject object = mObjects.get(id); + if (object == null) { + object = new SemanticsObject(); + object.id = id; + mObjects.put(id, object); + } + return object; + } + void handleTouchExplorationExit() { if (mHoveredObject != null) { sendAccessibilityEvent(mHoveredObject.id, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT); @@ -207,8 +231,7 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics if (mObjects.isEmpty()) { return; } - assert mObjects.containsKey(0); - SemanticObject newObject = getRootObject().hitTest(Math.round(x), Math.round(y)); + SemanticsObject newObject = getRootObject().hitTest(new float[]{ x, y, 0, 1 }); if (newObject != mHoveredObject) { // sending ENTER before EXIT is how Android wants it if (newObject != null) { @@ -221,57 +244,35 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics } } - @Override - public void updateSemanticsTree(SemanticsNode[] nodes) { - Set updatedObjects = new HashSet(); - Set removedObjects = new HashSet(); - for (SemanticsNode node : nodes) { - updateSemanticObject(node, updatedObjects, removedObjects); - sendAccessibilityEvent(node.id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED); + void updateSemantics(ByteBuffer buffer, String[] strings) { + buffer.order(ByteOrder.LITTLE_ENDIAN); + ArrayList updated = new ArrayList(); + while (buffer.hasRemaining()) { + int id = buffer.getInt(); + getOrCreateObject(id).updateWith(buffer, strings); + updated.add(id); } - for (SemanticObject object : removedObjects) { - if (!updatedObjects.contains(object)) { - removeSemanticObject(object, updatedObjects); - } - } - } - private SemanticObject updateSemanticObject(SemanticsNode node, - Set updatedObjects, - Set removedObjects) { - SemanticObject object = mObjects.get(node.id); - if (object == null) { - object = new SemanticObject(); - mObjects.put(node.id, object); + Set visitedObjects = new HashSet(); + SemanticsObject rootObject = getRootObject(); + if (rootObject != null) { + final float[] identity = new float[16]; + Matrix.setIdentityM(identity, 0); + rootObject.updateRecursively(identity, visitedObjects, false); } - object.updateWith(node); - updatedObjects.add(object); - if (node.children != null) { - if (node.children.length == 0) { - if (object.children != null) { - removedObjects.addAll(object.children); - } - object.children = null; - } else { - if (object.children == null) { - object.children = new ArrayList(node.children.length); - } else { - removedObjects.addAll(object.children); - object.children.clear(); - } - for (SemanticsNode childNode : node.children) { - SemanticObject childObject = updateSemanticObject(childNode, updatedObjects, removedObjects); - childObject.parent = object; - object.children.add(childObject); - } + + Iterator> it = mObjects.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + if (!visitedObjects.contains(entry.getKey())) { + willRemoveSemanticsObject(entry.getValue()); + it.remove(); } } - if (node.geometry != null) { - // has to be done after children are updated - // since they also get marked dirty - object.invalidateGlobalGeometry(); + + for (Integer id : updated) { + sendAccessibilityEvent(id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED); } - return object; } private void sendAccessibilityEvent(int virtualViewId, int eventType) { @@ -288,11 +289,10 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics } } - private void removeSemanticObject(SemanticObject object, Set updatedObjects) { + private void willRemoveSemanticsObject(SemanticsObject object) { assert mObjects.containsKey(object.id); assert mObjects.get(object.id) == object; object.parent = null; - mObjects.remove(object.id); if (mFocusedObject == object) { sendAccessibilityEvent(mFocusedObject.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED); mFocusedObject = null; @@ -300,190 +300,109 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics if (mHoveredObject == object) { mHoveredObject = null; } - if (object.children != null) { - for (SemanticObject child : object.children) { - if (!updatedObjects.contains(child)) { - assert child.parent == object; - removeSemanticObject(child, updatedObjects); - } - } - } } - void reset(SemanticsServer.Proxy newSemanticsServer) { + void reset() { mObjects.clear(); - sendAccessibilityEvent(mFocusedObject.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED); + if (mFocusedObject != null) + sendAccessibilityEvent(mFocusedObject.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED); mFocusedObject = null; mHoveredObject = null; - mSemanticsServer.close(); sendAccessibilityEvent(0, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED); - mSemanticsServer = newSemanticsServer; - mSemanticsServer.addSemanticsListener(this); } - private class SemanticObject { - SemanticObject() { } + private class SemanticsObject { + SemanticsObject() { } - void updateWith(SemanticsNode node) { - if (id == -1) { - id = node.id; - assert node.flags != null; - assert node.strings != null; - assert node.geometry != null; - assert node.children != null; - } - assert id == node.id; - if (node.flags != null) { - hasCheckedState = node.flags.hasCheckedState; - isChecked = node.flags.isChecked; - } - if (node.strings != null) { - label = node.strings.label; - } - if (node.geometry != null) { - transform = node.geometry.transform; - left = node.geometry.left; - top = node.geometry.top; - width = node.geometry.width; - height = node.geometry.height; - } - if (node.actions != null) { - canBeTapped = false; - canBeLongPressed = false; - canBeScrolledHorizontally = false; - canBeScrolledVertically = false; - for (int action : node.actions) { - switch (action) { - case SemanticAction.TAP: - canBeTapped = true; - break; - case SemanticAction.LONG_PRESS: - canBeLongPressed = true; - break; - case SemanticAction.SCROLL_LEFT: - canBeScrolledHorizontally = true; - break; - case SemanticAction.SCROLL_RIGHT: - canBeScrolledHorizontally = true; - break; - case SemanticAction.SCROLL_UP: - canBeScrolledVertically = true; - break; - case SemanticAction.SCROLL_DOWN: - canBeScrolledVertically = true; - break; - case SemanticAction.INCREASE: - // Not implemented. - break; - case SemanticAction.DECREASE: - // Not implemented. - break; - } - } - } - } - - // fields that we pass straight to the Android accessibility API int id = -1; - SemanticObject parent; - boolean canBeTapped; - boolean canBeLongPressed; - boolean canBeScrolledHorizontally; - boolean canBeScrolledVertically; - boolean hasCheckedState; - boolean isChecked; + + int actions; + int flags; String label; - List children; - // geometry, which we have to convert to global coordinates to send to Android - private float[] transform; // can be null, meaning identity transform private float left; private float top; - private float width; - private float height; + private float right; + private float bottom; + private float[] transform; - private boolean geometryDirty = true; - private void invalidateGlobalGeometry() { - if (geometryDirty) { - return; - } - geometryDirty = true; - // TODO(ianh): if we are the AccessibilityBridge.this.mFocusedObject - // then we may have to unfocus and refocus ourselves to get Android to update the focus rect - if (children != null) { - for (SemanticObject child : children) { - child.invalidateGlobalGeometry(); - } - } - } + SemanticsObject parent; + List children; - private float[] globalTransform; // cached transform from the root node to this node - private Rect globalRect; // cached Rect of bounds of this node in coordinate space of the root node + private boolean inverseTransformDirty = true; + private float[] inverseTransform; - private float[] getGlobalTransform() { - if (geometryDirty) { - if (parent == null) { - globalTransform = transform; - } else { - float[] parentTransform = parent.getGlobalTransform(); - if (transform == null) { - globalTransform = parentTransform; - } else if (parentTransform == null) { - globalTransform = transform; - } else { - globalTransform = new float[16]; - Matrix.multiplyMM(globalTransform, 0, transform, 0, parentTransform, 0); - } - } - } - return globalTransform; - } + private boolean globalGeometryDirty = true; + private float[] globalTransform; + private Rect globalRect; - private float[] transformPoint(float[] transform, float[] point) { - if (transform == null) - return point; // this is a 4-item array but the caller will ignore all but the first two items - float[] transformedPoint = new float[4]; - Matrix.multiplyMV(transformedPoint, 0, transform, 0, point, 0); - assert transformedPoint[2] == 0; - return new float[]{transformedPoint[0] / transformedPoint[3], - transformedPoint[1] / transformedPoint[3]}; - } + void updateWith(ByteBuffer buffer, String[] strings) { + actions = buffer.getInt(); + flags = buffer.getInt(); - private float min(float a, float b, float c, float d) { - return Math.min(a, Math.min(b, Math.min(c, d))); + final int stringIndex = buffer.getInt(); + if (stringIndex == -1) + label = null; + else + label = strings[stringIndex]; + + left = buffer.getFloat(); + top = buffer.getFloat(); + right = buffer.getFloat(); + bottom = buffer.getFloat(); + + if (transform == null) + transform = new float[16]; + for (int i = 0; i < 16; ++i) + transform[i] = buffer.getFloat(); + inverseTransformDirty = true; + globalGeometryDirty = true; + + final int childCount = buffer.getInt(); + if (childCount == 0) { + children = null; + } else { + if (children == null) + children = new ArrayList(childCount); + else + children.clear(); + + for (int i = 0; i < childCount; ++i) { + SemanticsObject child = getOrCreateObject(buffer.getInt()); + children.add(child); + child.parent = this; + } + } } - private float max(float a, float b, float c, float d) { - return Math.max(a, Math.max(b, Math.max(c, d))); + private void ensureInverseTransform() { + if (!inverseTransformDirty) + return; + inverseTransformDirty = false; + if (inverseTransform == null) + inverseTransform = new float[16]; + if (!Matrix.invertM(inverseTransform, 0, transform, 0)) + Arrays.fill(inverseTransform, 0); } Rect getGlobalRect() { - if (geometryDirty) { - float[] transform = getGlobalTransform(); - float[] point1 = transformPoint(transform, new float[]{left, top, 0, 1}); - float[] point2 = transformPoint(transform, new float[]{left + width, top, 0, 1}); - float[] point3 = transformPoint(transform, new float[]{left + width, top + height, 0, 1}); - float[] point4 = transformPoint(transform, new float[]{left, top + height, 0, 1}); - // TODO(ianh): Scaling here is a hack to work around #1360. - float scale = mOwner.getDevicePixelRatio(); - globalRect = new Rect( - Math.round(min(point1[0], point2[0], point3[0], point4[0]) * scale), - Math.round(min(point1[1], point2[1], point3[1], point4[1]) * scale), - Math.round(max(point1[0], point2[0], point3[0], point4[0]) * scale), - Math.round(max(point1[1], point2[1], point3[1], point4[1]) * scale) - ); - } + assert !globalGeometryDirty; return globalRect; } - SemanticObject hitTest(int x, int y) { - Rect rect = getGlobalRect(); - if (!rect.contains(x, y)) + SemanticsObject hitTest(float[] point) { + final float w = point[3]; + final float x = point[0] / w; + final float y = point[1] / w; + if (x < left || x >= right || y < top || y >= bottom) return null; if (children != null) { - for (int index = children.size()-1; index >= 0; index -= 1) { - SemanticObject child = children.get(index); - SemanticObject result = child.hitTest(x, y); + final float[] transformedPoint = new float[4]; + for (int i = children.size() - 1; i >= 0; i -= 1) { + final SemanticsObject child = children.get(i); + child.ensureInverseTransform(); + Matrix.multiplyMV(transformedPoint, 0, child.inverseTransform, 0, point, 0); + final SemanticsObject result = child.hitTest(transformedPoint); if (result != null) { return result; } @@ -491,12 +410,81 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements Semantics } return this; } - } - @Override - public void close() {} + void updateRecursively(float[] ancestorTransform, Set visitedObjects, boolean forceUpdate) { + visitedObjects.add(this); - @Override - public void onConnectionError(MojoException e) {} + if (globalGeometryDirty) + forceUpdate = true; + + if (forceUpdate) { + if (globalTransform == null) + globalTransform = new float[16]; + Matrix.multiplyMM(globalTransform, 0, transform, 0, ancestorTransform, 0); + + final float[] sample = new float[4]; + sample[2] = 0; + sample[3] = 1; + + final float[] point1 = new float[4]; + final float[] point2 = new float[4]; + final float[] point3 = new float[4]; + final float[] point4 = new float[4]; + + sample[0] = left; + sample[1] = top; + transformPoint(point1, globalTransform, sample); + + sample[0] = right; + sample[1] = top; + transformPoint(point2, globalTransform, sample); + + sample[0] = right; + sample[1] = bottom; + transformPoint(point3, globalTransform, sample); + sample[0] = left; + sample[1] = bottom; + transformPoint(point4, globalTransform, sample); + + if (globalRect == null) + globalRect = new Rect(); + + globalRect.set( + Math.round(min(point1[0], point2[0], point3[0], point4[0])), + Math.round(min(point1[1], point2[1], point3[1], point4[1])), + Math.round(max(point1[0], point2[0], point3[0], point4[0])), + Math.round(max(point1[1], point2[1], point3[1], point4[1])) + ); + + globalGeometryDirty = false; + } + + assert globalTransform != null; + assert globalRect != null; + + if (children != null) { + for (int i = 0; i < children.size(); ++i) { + children.get(i).updateRecursively(globalTransform, visitedObjects, forceUpdate); + } + } + } + + private void transformPoint(float[] result, float[] transform, float[] point) { + Matrix.multiplyMV(result, 0, transform, 0, point, 0); + final float w = result[3]; + result[0] /= w; + result[1] /= w; + result[2] /= w; + result[3] = 0; + } + + private float min(float a, float b, float c, float d) { + return Math.min(a, Math.min(b, Math.min(c, d))); + } + + private float max(float a, float b, float c, float d) { + return Math.max(a, Math.max(b, Math.max(c, d))); + } + } } diff --git a/shell/platform/android/io/flutter/view/FlutterView.java b/shell/platform/android/io/flutter/view/FlutterView.java index 50badcb9d..27e005f37 100644 --- a/shell/platform/android/io/flutter/view/FlutterView.java +++ b/shell/platform/android/io/flutter/view/FlutterView.java @@ -46,7 +46,6 @@ import org.chromium.mojom.editing.Keyboard; import org.chromium.mojom.flutter.platform.ApplicationMessages; import org.chromium.mojom.mojo.ServiceProvider; import org.chromium.mojom.raw_keyboard.RawKeyboardService; -import org.chromium.mojom.semantics.SemanticsServer; import org.chromium.mojom.sky.AppLifecycleState; import org.chromium.mojom.sky.ServicesData; import org.chromium.mojom.sky.SkyEngine; @@ -578,7 +577,8 @@ public class FlutterView extends SurfaceView private static native Bitmap nativeGetBitmap(long nativePlatformViewAndroid); private static native void nativeDispatchPointerDataPacket(long nativePlatformViewAndroid, ByteBuffer buffer, int position); - + private static native void nativeDispatchSemanticsAction(long nativePlatformViewAndroid, int id, int action); + private static native void nativeSetSemanticsEnabled(long nativePlatformViewAndroid, boolean enabled); private static native void nativeInvokePlatformMessageResponseCallback(long nativePlatformViewAndroid, int responseId, String buffer); @CalledByNative @@ -603,11 +603,21 @@ public class FlutterView extends SurfaceView nativeInvokePlatformMessageResponseCallback(mNativePlatformView, responseId, null); } + @CalledByNative + private void updateSemantics(ByteBuffer buffer, String[] strings) { + if (mAccessibilityNodeProvider != null) + mAccessibilityNodeProvider.updateSemantics(buffer, strings); + } + // ACCESSIBILITY private boolean mAccessibilityEnabled = false; private boolean mTouchExplorationEnabled = false; + protected void dispatchSemanticsAction(int id, int action) { + nativeDispatchSemanticsAction(mNativePlatformView, id, action); + } + @Override protected void onAttachedToWindow() { super.onAttachedToWindow(); @@ -669,21 +679,14 @@ public class FlutterView extends SurfaceView void ensureAccessibilityEnabled() { if (mAccessibilityNodeProvider == null) { - mAccessibilityNodeProvider = new AccessibilityBridge(this, createSemanticsServer()); + mAccessibilityNodeProvider = new AccessibilityBridge(this); + nativeSetSemanticsEnabled(mNativePlatformView, true); } } - private SemanticsServer.Proxy createSemanticsServer() { - Core core = CoreImpl.getInstance(); - Pair> server = - SemanticsServer.MANAGER.getInterfaceRequest(core); - mDartServiceProvider.connectToService(SemanticsServer.MANAGER.getName(), server.second.passHandle()); - return server.first; - } - void resetAccessibilityTree() { if (mAccessibilityNodeProvider != null) { - mAccessibilityNodeProvider.reset(createSemanticsServer()); + mAccessibilityNodeProvider.reset(); } } diff --git a/shell/platform/android/platform_view_android.cc b/shell/platform/android/platform_view_android.cc index 4edc27cc0..34f8276fd 100644 --- a/shell/platform/android/platform_view_android.cc +++ b/shell/platform/android/platform_view_android.cc @@ -9,9 +9,12 @@ #include #include #include + #include #include + #include "base/android/jni_android.h" +#include "base/android/jni_array.h" #include "base/android/jni_string.h" #include "base/bind.h" #include "base/location.h" @@ -29,7 +32,7 @@ namespace shell { PlatformViewAndroid::PlatformViewAndroid() - : PlatformView(std::make_unique()), weak_factory_(this) {} + : PlatformView(std::make_unique()) {} PlatformViewAndroid::~PlatformViewAndroid() = default; @@ -159,6 +162,20 @@ void PlatformViewAndroid::HandlePlatformMessage( } } +void PlatformViewAndroid::DispatchSemanticsAction(JNIEnv* env, + jobject obj, + jint id, + jint action) { + PlatformView::DispatchSemanticsAction( + id, static_cast(action)); +} + +void PlatformViewAndroid::SetSemanticsEnabled(JNIEnv* env, + jobject obj, + jboolean enabled) { + PlatformView::SetSemanticsEnabled(enabled); +} + void PlatformViewAndroid::ReleaseSurface() { if (surface_gl_) { NotifyDestroyed(); @@ -166,10 +183,6 @@ void PlatformViewAndroid::ReleaseSurface() { } } -ftl::WeakPtr PlatformViewAndroid::GetWeakViewPtr() { - return weak_factory_.GetWeakPtr(); -} - bool PlatformViewAndroid::ResourceContextMakeCurrent() { return surface_gl_ ? surface_gl_->GLOffscreenContextMakeCurrent() : false; } @@ -184,6 +197,56 @@ void PlatformViewAndroid::Resize(const SkISize& size) { } } +void PlatformViewAndroid::UpdateSemantics( + std::vector update) { + constexpr size_t kBytesPerNode = 25 * sizeof(int32_t); + constexpr size_t kBytesPerChild = sizeof(int32_t); + + JNIEnv* env = base::android::AttachCurrentThread(); + { + base::android::ScopedJavaLocalRef view = flutter_view_.get(env); + if (view.is_null()) + return; + + size_t num_bytes = 0; + for (const blink::SemanticsNode& node : update) { + num_bytes += kBytesPerNode; + num_bytes += node.children.size() * kBytesPerChild; + } + + std::vector buffer(num_bytes); + int32_t* buffer_int32 = reinterpret_cast(&buffer[0]); + float* buffer_float32 = reinterpret_cast(&buffer[0]); + + std::vector strings; + size_t position = 0; + for (const blink::SemanticsNode& node : update) { + buffer_int32[position++] = node.id; + buffer_int32[position++] = node.flags; + buffer_int32[position++] = node.actions; + if (node.label.empty()) { + buffer_int32[position++] = -1; + } else { + buffer_int32[position++] = strings.size(); + strings.push_back(node.label); + } + buffer_float32[position++] = node.rect.left(); + buffer_float32[position++] = node.rect.top(); + buffer_float32[position++] = node.rect.right(); + buffer_float32[position++] = node.rect.bottom(); + node.transform.asColMajorf(&buffer_float32[position]); + position += 16; + buffer_int32[position++] = node.children.size(); + for (int32_t child : node.children) + buffer_int32[position++] = child; + } + + Java_FlutterView_updateSemantics( + env, view.obj(), env->NewDirectByteBuffer(buffer.data(), buffer.size()), + base::android::ToJavaArrayOfStrings(env, strings).obj()); + } +} + void PlatformViewAndroid::RunFromSource(const std::string& main, const std::string& packages, const std::string& assets_directory) { diff --git a/shell/platform/android/platform_view_android.h b/shell/platform/android/platform_view_android.h index 43a109443..5231394f5 100644 --- a/shell/platform/android/platform_view_android.h +++ b/shell/platform/android/platform_view_android.h @@ -44,20 +44,24 @@ class PlatformViewAndroid : public PlatformView { jint response_id, jstring response); + void DispatchSemanticsAction(JNIEnv* env, jobject obj, jint id, jint action); + + void SetSemanticsEnabled(JNIEnv* env, jobject obj, jboolean enabled); + base::android::ScopedJavaLocalRef GetBitmap(JNIEnv* env, jobject obj); - ftl::WeakPtr GetWeakViewPtr() override; - bool ResourceContextMakeCurrent() override; - virtual SkISize GetSize(); + SkISize GetSize() override; + + void Resize(const SkISize& size) override; - virtual void Resize(const SkISize& size); + void UpdateSemantics(std::vector update) override; - virtual void RunFromSource(const std::string& main, - const std::string& packages, - const std::string& assets_directory); + void RunFromSource(const std::string& main, + const std::string& packages, + const std::string& assets_directory) override; void set_flutter_view(const JavaObjectWeakGlobalRef& flutter_view) { flutter_view_ = flutter_view; @@ -66,7 +70,6 @@ class PlatformViewAndroid : public PlatformView { private: std::unique_ptr surface_gl_; JavaObjectWeakGlobalRef flutter_view_; - ftl::WeakPtrFactory weak_factory_; // We use id 0 to mean that no response is expected. int next_response_id_ = 1; diff --git a/shell/platform/darwin/desktop/platform_view_mac.h b/shell/platform/darwin/desktop/platform_view_mac.h index c4c7a23ea..d2bda9180 100644 --- a/shell/platform/darwin/desktop/platform_view_mac.h +++ b/shell/platform/darwin/desktop/platform_view_mac.h @@ -25,8 +25,6 @@ class PlatformViewMac : public PlatformView, public GPUSurfaceGLDelegate { sky::SkyEnginePtr& engineProxy(); - ftl::WeakPtr GetWeakViewPtr() override; - bool GLContextMakeCurrent() override; bool GLContextClearCurrent() override; @@ -45,7 +43,6 @@ class PlatformViewMac : public PlatformView, public GPUSurfaceGLDelegate { base::scoped_nsobject opengl_view_; base::scoped_nsobject resource_loading_context_; sky::SkyEnginePtr sky_engine_; - ftl::WeakPtrFactory weak_factory_; bool IsValid() const; diff --git a/shell/platform/darwin/desktop/platform_view_mac.mm b/shell/platform/darwin/desktop/platform_view_mac.mm index ea42c17c8..42186ab51 100644 --- a/shell/platform/darwin/desktop/platform_view_mac.mm +++ b/shell/platform/darwin/desktop/platform_view_mac.mm @@ -23,8 +23,7 @@ PlatformViewMac::PlatformViewMac(NSOpenGLView* gl_view) opengl_view_([gl_view retain]), resource_loading_context_([[NSOpenGLContext alloc] initWithFormat:gl_view.pixelFormat - shareContext:gl_view.openGLContext]), - weak_factory_(this) { + shareContext:gl_view.openGLContext]) { NSArray* paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES); if (paths.count > 0) { @@ -93,10 +92,6 @@ sky::SkyEnginePtr& PlatformViewMac::engineProxy() { return sky_engine_; } -ftl::WeakPtr PlatformViewMac::GetWeakViewPtr() { - return weak_factory_.GetWeakPtr(); -} - intptr_t PlatformViewMac::GLContextFBO() const { // Default window bound framebuffer FBO 0. return 0; diff --git a/shell/platform/darwin/ios/BUILD.gn b/shell/platform/darwin/ios/BUILD.gn index 90eb939d3..7fc0a0792 100644 --- a/shell/platform/darwin/ios/BUILD.gn +++ b/shell/platform/darwin/ios/BUILD.gn @@ -44,6 +44,7 @@ shared_library("flutter_framework_dylib") { deps = [ "//base:base", "//dart/runtime:libdart", + "//flutter/lib/ui", "//flutter/services/activity", "//flutter/services/editing", "//flutter/services/engine:interfaces", diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm index a863e640f..c181e8aad 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm @@ -364,11 +364,11 @@ static inline PointerChangeMapperPhase PointerChangePhaseFromUITouchPhase( // There doesn't appear to be any way to determine whether the accessibility // inspector is enabled on the simulator. We conservatively always turn on the // accessibility bridge in the simulator. - bool enable = true; + bool enabled = true; #else - bool enable = UIAccessibilityIsVoiceOverRunning(); + bool enabled = UIAccessibilityIsVoiceOverRunning(); #endif - _platformView->ToggleAccessibility(self.view, enable); + _platformView->ToggleAccessibility(self.view, enabled); } #pragma mark - Locale updates diff --git a/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h b/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h index 0f8be847d..4e3467ae8 100644 --- a/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h +++ b/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h @@ -6,16 +6,13 @@ #define SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_ACCESSIBILITY_BRIDGE_H_ #include -#include #include +#include +#include -#include "flutter/services/semantics/semantics.mojom.h" +#include "flutter/lib/ui/semantics/semantics_node.h" #include "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h" -#include "flutter/sky/engine/platform/geometry/FloatRect.h" #include "lib/ftl/macros.h" -#include "mojo/public/cpp/bindings/array.h" -#include "mojo/public/cpp/bindings/strong_binding.h" -#include "mojo/public/interfaces/application/service_provider.mojom.h" #include "third_party/skia/include/core/SkMatrix44.h" #include "third_party/skia/include/core/SkRect.h" @@ -23,50 +20,47 @@ namespace shell { class AccessibilityBridge; } // namespace shell -@interface SemanticObject : NSObject +@interface SemanticsObject : NSObject /** * The globally unique identifier for this node. */ -@property(nonatomic, readonly) uint32_t uid; +@property(nonatomic, readonly) int32_t uid; /** * The parent of this node in the node tree. Will be nil for the root node and * during transient state changes. */ -@property(nonatomic, assign) SemanticObject* parent; +@property(nonatomic, assign) SemanticsObject* parent; - (instancetype)init __attribute__((unavailable("Use initWithBridge instead"))); - (instancetype)initWithBridge:(shell::AccessibilityBridge*)bridge - uid:(uint32_t)uid NS_DESIGNATED_INITIALIZER; + uid:(int32_t)uid NS_DESIGNATED_INITIALIZER; @end namespace shell { +class PlatformViewIOS; -class AccessibilityBridge final : public semantics::SemanticsListener { +class AccessibilityBridge final { public: - AccessibilityBridge(UIView*, mojo::ServiceProvider*); - ~AccessibilityBridge() override; + AccessibilityBridge(UIView* view, PlatformViewIOS* platform_view); + ~AccessibilityBridge(); - void UpdateSemanticsTree(mojo::Array) override; + void UpdateSemantics(std::vector nodes); + void DispatchSemanticsAction(int32_t id, blink::SemanticsAction action); - UIView* view() { return view_; } - semantics::SemanticsServer* server() { return semantics_server_.get(); } + UIView* view() const { return view_; } private: - SemanticObject* UpdateSemanticObject( - const semantics::SemanticsNodePtr& node, - std::set* updated_objects, - std::set* removed_objects); - void RemoveSemanticObject(SemanticObject* node, - std::set* updated_objects); + SemanticsObject* GetOrCreateObject(int32_t id); + void VisitObjectsRecursively(SemanticsObject* object, + std::unordered_set* visited_objects); + void ReleaseObjects(const std::unordered_map& objects); UIView* view_; - semantics::SemanticsServerPtr semantics_server_; - std::unordered_map objects_; - - mojo::Binding binding_; + PlatformViewIOS* platform_view_; + std::unordered_map objects_; FTL_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge); }; diff --git a/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm b/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm index 771b0dbd3..454fee410 100644 --- a/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm +++ b/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm @@ -4,45 +4,42 @@ #include "flutter/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h" -#import #include +#include + +#import -#include "base/logging.h" -#include "mojo/public/cpp/application/connect.h" +#include "lib/ftl/logging.h" +#include "flutter/shell/platform/darwin/ios/platform_view_ios.h" namespace { -constexpr uint32_t kRootNodeId = 0; +constexpr int32_t kRootNodeId = 0; -// Contains better abstractions than the raw Mojo data structure -struct Geometry { - Geometry& operator=(const semantics::SemanticGeometryPtr& other) { - if (!other->transform.is_null()) { - transform.setColMajorf(other->transform.data()); - } - rect.setXYWH(other->left, other->top, other->width, other->height); - return *this; +blink::SemanticsAction GetSemanticsActionForScrollDirection( + UIAccessibilityScrollDirection direction) { + switch (direction) { + case UIAccessibilityScrollDirectionRight: + case UIAccessibilityScrollDirectionPrevious: // TODO(abarth): Support RTL. + return blink::SemanticsAction::kScrollRight; + case UIAccessibilityScrollDirectionLeft: + case UIAccessibilityScrollDirectionNext: // TODO(abarth): Support RTL. + return blink::SemanticsAction::kScrollLeft; + case UIAccessibilityScrollDirectionUp: + return blink::SemanticsAction::kScrollUp; + case UIAccessibilityScrollDirectionDown: + return blink::SemanticsAction::kScrollDown; } - - SkMatrix44 transform = SkMatrix44(SkMatrix44::kIdentity_Constructor); - SkRect rect; -}; + FTL_DCHECK(false); // Unreachable + return blink::SemanticsAction::kScrollDown; +} } // namespace -@implementation SemanticObject { +@implementation SemanticsObject { shell::AccessibilityBridge* _bridge; - - semantics::SemanticFlagsPtr _flags; - semantics::SemanticStringsPtr _strings; - Geometry _geometry; - bool _canBeTapped; - bool _canBeLongPressed; - bool _canBeScrolledHorizontally; - bool _canBeScrolledVertically; - bool _canBeAdjusted; - - std::vector _children; + blink::SemanticsNode _node; + std::vector _children; } #pragma mark - Override base class designated initializers @@ -57,9 +54,9 @@ struct Geometry { #pragma mark - Designated initializers - (instancetype)initWithBridge:(shell::AccessibilityBridge*)bridge - uid:(uint32_t)uid { - DCHECK(bridge != nil) << "bridge must be set"; - DCHECK(uid >= kRootNodeId); + uid:(int32_t)uid { + FTL_DCHECK(bridge != nil) << "bridge must be set"; + FTL_DCHECK(uid >= kRootNodeId); self = [super init]; if (self) { @@ -72,52 +69,11 @@ struct Geometry { #pragma mark - Semantic object methods -- (void)updateWith:(const semantics::SemanticsNodePtr&)node { - DCHECK(_uid == node->id); - - if (!node->flags.is_null()) { - _flags = node->flags.Pass(); - } - - if (!node->strings.is_null()) { - _strings = node->strings.Pass(); - } - - if (!node->geometry.is_null()) { - _geometry = node->geometry; - } - - if (!node->actions.is_null()) { - _canBeTapped = false; - _canBeLongPressed = false; - _canBeScrolledHorizontally = false; - _canBeScrolledVertically = false; - for (int action : node->actions) { - switch (static_cast(action)) { - case semantics::SemanticAction::TAP: - _canBeTapped = true; - break; - case semantics::SemanticAction::LONG_PRESS: - _canBeLongPressed = true; - break; - case semantics::SemanticAction::SCROLL_LEFT: - case semantics::SemanticAction::SCROLL_RIGHT: - _canBeScrolledHorizontally = true; - break; - case semantics::SemanticAction::SCROLL_UP: - case semantics::SemanticAction::SCROLL_DOWN: - _canBeScrolledVertically = true; - break; - case semantics::SemanticAction::INCREASE: - case semantics::SemanticAction::DECREASE: - _canBeAdjusted = true; - break; - } - } - } +- (void)setSemanticsNode:(const blink::SemanticsNode*)node { + _node = *node; } -- (std::vector*)children { +- (std::vector*)children { return &_children; } @@ -133,39 +89,40 @@ struct Geometry { // Note: hit detection will only apply to elements that report // -isAccessibilityElement of YES. The framework will continue scanning the // entire element tree looking for such a hit. - return _canBeTapped || _children.empty(); + return _node.HasAction(blink::SemanticsAction::kTap) || _children.empty(); } - (NSString*)accessibilityLabel { - if (_strings.is_null() || _strings->label.get().empty()) { + if (_node.label.empty()) { return nil; } - return @(_strings->label.data()); + return @(_node.label.data()); } - (UIAccessibilityTraits)accessibilityTraits { UIAccessibilityTraits traits = UIAccessibilityTraitNone; - if (_canBeTapped) { + if (_node.HasAction(blink::SemanticsAction::kTap)) { traits |= UIAccessibilityTraitButton; } - if (_canBeAdjusted) { + if (_node.HasAction(blink::SemanticsAction::kIncrease) + || _node.HasAction(blink::SemanticsAction::kDecrease)) { traits |= UIAccessibilityTraitAdjustable; } return traits; } - (CGRect)accessibilityFrame { - SkMatrix44 globalTransform = _geometry.transform; - for (SemanticObject* parent = _parent; parent; parent = parent.parent) { - globalTransform = globalTransform * parent->_geometry.transform; + SkMatrix44 globalTransform = _node.transform; + for (SemanticsObject* parent = _parent; parent; parent = parent.parent) { + globalTransform = globalTransform * parent->_node.transform; } SkPoint quad[4]; - _geometry.rect.toQuad(quad); + _node.rect.toQuad(quad); for (auto& point : quad) { SkScalar vector[4] = {point.x(), point.y(), 0, 1}; globalTransform.mapScalars(vector); - point.set(vector[0], vector[1]); + point.set(vector[0] / vector[3], vector[1] / vector[3]); } SkRect rect; rect.set(quad, 4); @@ -210,61 +167,22 @@ struct Geometry { } - (void)accessibilityIncrement { - if (_canBeAdjusted) { - _bridge->server()->PerformAction(_uid, semantics::SemanticAction::INCREASE); + if (_node.HasAction(blink::SemanticsAction::kIncrease)) { + _bridge->DispatchSemanticsAction(_uid, blink::SemanticsAction::kIncrease); } } - (void)accessibilityDecrement { - if (_canBeAdjusted) { - _bridge->server()->PerformAction(_uid, semantics::SemanticAction::DECREASE); + if (_node.HasAction(blink::SemanticsAction::kDecrease)) { + _bridge->DispatchSemanticsAction(_uid, blink::SemanticsAction::kDecrease); } } - (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction { - BOOL canBeScrolled = NO; - switch (direction) { - case UIAccessibilityScrollDirectionRight: - case UIAccessibilityScrollDirectionLeft: - canBeScrolled = _canBeScrolledHorizontally; - break; - case UIAccessibilityScrollDirectionUp: - case UIAccessibilityScrollDirectionDown: - canBeScrolled = _canBeScrolledVertically; - break; - default: - // Note: page turning of reading content is not currently supported - // (UIAccessibilityScrollDirectionNext, - // UIAccessibilityScrollDirectionPrevious) - canBeScrolled = NO; - break; - } - - if (!canBeScrolled) { + blink::SemanticsAction action = GetSemanticsActionForScrollDirection(direction); + if (_node.HasAction(action)) return NO; - } - - switch (direction) { - case UIAccessibilityScrollDirectionRight: - _bridge->server()->PerformAction(_uid, - semantics::SemanticAction::SCROLL_RIGHT); - break; - case UIAccessibilityScrollDirectionLeft: - _bridge->server()->PerformAction(_uid, - semantics::SemanticAction::SCROLL_LEFT); - break; - case UIAccessibilityScrollDirectionUp: - _bridge->server()->PerformAction(_uid, - semantics::SemanticAction::SCROLL_UP); - break; - case UIAccessibilityScrollDirectionDown: - _bridge->server()->PerformAction(_uid, - semantics::SemanticAction::SCROLL_DOWN); - break; - default: - DCHECK(false) << "Unsupported scroll direction: " << direction; - } - + _bridge->DispatchSemanticsAction(_uid, action); // TODO(tvolkert): provide meaningful string (e.g. "page 2 of 5") UIAccessibilityPostNotification(UIAccessibilityPageScrolledNotification, nil); return YES; @@ -287,38 +205,46 @@ struct Geometry { namespace shell { AccessibilityBridge::AccessibilityBridge(UIView* view, - mojo::ServiceProvider* serviceProvider) - : view_(view), binding_(this) { - mojo::ConnectToService(serviceProvider, mojo::GetProxy(&semantics_server_)); - mojo::InterfaceHandle listener; - binding_.Bind(&listener); - semantics_server_->AddSemanticsListener(listener.Pass()); -} + PlatformViewIOS* platform_view) + : view_(view), platform_view_(platform_view) {} AccessibilityBridge::~AccessibilityBridge() { - for (const auto& entry : objects_) { - SemanticObject* object = entry.second; - [object neuter]; - [object release]; - } + ReleaseObjects(objects_); + objects_.clear(); } -void AccessibilityBridge::UpdateSemanticsTree( - mojo::Array nodes) { - std::set updated_objects; - std::set removed_objects; - - for (const semantics::SemanticsNodePtr& node : nodes) { - UpdateSemanticObject(node, &updated_objects, &removed_objects); +void AccessibilityBridge::UpdateSemantics(std::vector nodes) { + for (const blink::SemanticsNode& node : nodes) { + SemanticsObject* object = GetOrCreateObject(node.id); + [object setSemanticsNode:&node]; + const size_t childrenCount = node.children.size(); + auto& children = *[object children]; + children.resize(childrenCount); + for (size_t i = 0; i < childrenCount; ++i) { + SemanticsObject* child = GetOrCreateObject(node.children[i]); + child.parent = object; + children[i] = child; + } } - for (SemanticObject* object : removed_objects) { - if (!updated_objects.count(object)) { - RemoveSemanticObject(object, &updated_objects); - } + SemanticsObject* root = objects_[kRootNodeId]; + + std::unordered_set visited_objects; + if (root) + VisitObjectsRecursively(root, &visited_objects); + + std::unordered_map doomed_objects; + doomed_objects.swap(objects_); + for (int uid : visited_objects) { + auto it = doomed_objects.find(uid); + objects_.insert(*it); + doomed_objects.erase(it); + // TODO(abarth): Use extract once we're at C++17. } - SemanticObject* root = objects_[kRootNodeId]; + ReleaseObjects(doomed_objects); + doomed_objects.clear(); + if (root) { if (!view_.accessibilityElements) { view_.accessibilityElements = @[ root ]; @@ -330,46 +256,36 @@ void AccessibilityBridge::UpdateSemanticsTree( nil); } -SemanticObject* AccessibilityBridge::UpdateSemanticObject( - const semantics::SemanticsNodePtr& node, - std::set* updated_objects, - std::set* removed_objects) { - SemanticObject* object = objects_[node->id]; +void AccessibilityBridge::DispatchSemanticsAction( + int32_t uid, + blink::SemanticsAction action) { + platform_view_->DispatchSemanticsAction(uid, action); +} + +SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid) { + SemanticsObject* object = objects_[uid]; if (!object) { - object = [[SemanticObject alloc] initWithBridge:this uid:node->id]; - objects_[node->id] = object; - } - [object updateWith:node]; - updated_objects->insert(object); - if (!node->children.is_null()) { - std::vector* children = [object children]; - removed_objects->insert(children->begin(), children->end()); - children->clear(); - children->reserve(node->children.size()); - for (const auto& child_node : node->children) { - SemanticObject* child_object = - UpdateSemanticObject(child_node, updated_objects, removed_objects); - child_object.parent = object; - children->push_back(child_object); - } + object = [[SemanticsObject alloc] initWithBridge:this uid:uid]; + objects_[uid] = object; } return object; } -void AccessibilityBridge::RemoveSemanticObject( - SemanticObject* object, - std::set* updated_objects) { - DCHECK(objects_[object.uid] == object); - objects_.erase(object.uid); - for (SemanticObject* child : *[object children]) { - if (!updated_objects->count(child)) { - DCHECK(child.parent == object); - child.parent = nil; - RemoveSemanticObject(child, updated_objects); - } +void AccessibilityBridge::VisitObjectsRecursively( + SemanticsObject* object, + std::unordered_set* visited_objects) { + visited_objects->insert(object.uid); + for (SemanticsObject* child : *[object children]) + VisitObjectsRecursively(child, visited_objects); +} + +void AccessibilityBridge::ReleaseObjects( + const std::unordered_map& objects) { + for (const auto& entry : objects) { + SemanticsObject* object = entry.second; + [object neuter]; + [object release]; } - [object neuter]; - [object release]; } } // namespace shell diff --git a/shell/platform/darwin/ios/platform_view_ios.h b/shell/platform/darwin/ios/platform_view_ios.h index b90ea89dc..862ccb73a 100644 --- a/shell/platform/darwin/ios/platform_view_ios.h +++ b/shell/platform/darwin/ios/platform_view_ios.h @@ -29,7 +29,7 @@ class PlatformViewIOS : public PlatformView, public GPUSurfaceGLDelegate { ~PlatformViewIOS() override; - void ToggleAccessibility(UIView* view, bool enable); + void ToggleAccessibility(UIView* view, bool enabled); void ConnectToEngineAndSetupServices(); @@ -39,8 +39,6 @@ class PlatformViewIOS : public PlatformView, public GPUSurfaceGLDelegate { ApplicationMessagesImpl& AppMessageReceiver(); - ftl::WeakPtr GetWeakViewPtr() override; - bool ResourceContextMakeCurrent() override; bool GLContextMakeCurrent() override; @@ -55,6 +53,8 @@ class PlatformViewIOS : public PlatformView, public GPUSurfaceGLDelegate { const std::string& packages, const std::string& assets_directory) override; + void UpdateSemantics(std::vector update) override; + private: std::unique_ptr context_; sky::SkyEnginePtr engine_; @@ -62,7 +62,6 @@ class PlatformViewIOS : public PlatformView, public GPUSurfaceGLDelegate { flutter::platform::ApplicationMessagesPtr app_message_sender_; ApplicationMessagesImpl app_message_receiver_; std::unique_ptr accessibility_bridge_; - ftl::WeakPtrFactory weak_factory_; void SetupAndLoadFromSource(const std::string& main, const std::string& packages, diff --git a/shell/platform/darwin/ios/platform_view_ios.mm b/shell/platform/darwin/ios/platform_view_ios.mm index 498796e3c..8b0cae237 100644 --- a/shell/platform/darwin/ios/platform_view_ios.mm +++ b/shell/platform/darwin/ios/platform_view_ios.mm @@ -273,8 +273,7 @@ class IOSGLContext { PlatformViewIOS::PlatformViewIOS(CAEAGLLayer* layer) : PlatformView(std::make_unique()), - context_(std::make_unique(surface_config_, layer)), - weak_factory_(this) { + context_(std::make_unique(surface_config_, layer)) { NSArray* paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES); shell::Shell::Shared().tracing_controller().set_traces_base_path( @@ -295,15 +294,16 @@ shell::ApplicationMessagesImpl& PlatformViewIOS::AppMessageReceiver() { return app_message_receiver_; } -void PlatformViewIOS::ToggleAccessibility(UIView* view, bool enable) { - if (enable) { - if (!accessibility_bridge_ && dart_services_.get() != nullptr) { +void PlatformViewIOS::ToggleAccessibility(UIView* view, bool enabled) { + if (enabled) { + if (!accessibility_bridge_) { accessibility_bridge_.reset( - new shell::AccessibilityBridge(view, dart_services_.get())); + new shell::AccessibilityBridge(view, this)); } } else { accessibility_bridge_ = nullptr; } + SetSemanticsEnabled(enabled); } void PlatformViewIOS::ConnectToEngineAndSetupServices() { @@ -347,10 +347,6 @@ void PlatformViewIOS::SetupAndLoadFromSource( engine_->RunFromFile(main, packages, assets_directory); } -ftl::WeakPtr PlatformViewIOS::GetWeakViewPtr() { - return weak_factory_.GetWeakPtr(); -} - bool PlatformViewIOS::ResourceContextMakeCurrent() { return context_ != nullptr ? context_->ResourceMakeCurrent() : false; } @@ -387,4 +383,10 @@ void PlatformViewIOS::RunFromSource(const std::string& main, delete latch; } +void PlatformViewIOS::UpdateSemantics( + std::vector update) { + if (accessibility_bridge_) + accessibility_bridge_->UpdateSemantics(std::move(update)); +} + } // namespace shell diff --git a/shell/platform/linux/platform_view_glfw.cc b/shell/platform/linux/platform_view_glfw.cc index b4b58190c..0618034ed 100644 --- a/shell/platform/linux/platform_view_glfw.cc +++ b/shell/platform/linux/platform_view_glfw.cc @@ -18,8 +18,7 @@ PlatformViewGLFW::PlatformViewGLFW() : PlatformView(std::make_unique()), valid_(false), glfw_window_(nullptr), - buttons_(0), - weak_factory_(this) { + buttons_(0) { if (!glfwInit()) { return; } @@ -78,10 +77,6 @@ bool PlatformViewGLFW::IsValid() const { return valid_; } -ftl::WeakPtr PlatformViewGLFW::GetWeakViewPtr() { - return weak_factory_.GetWeakPtr(); -} - intptr_t PlatformViewGLFW::GLContextFBO() const { // The default window bound FBO. return 0; diff --git a/shell/platform/linux/platform_view_glfw.h b/shell/platform/linux/platform_view_glfw.h index ff1936878..79513f6e3 100644 --- a/shell/platform/linux/platform_view_glfw.h +++ b/shell/platform/linux/platform_view_glfw.h @@ -26,8 +26,6 @@ class PlatformViewGLFW : public PlatformView, public GPUSurfaceGLDelegate { bool IsValid() const; - ftl::WeakPtr GetWeakViewPtr() override; - bool ResourceContextMakeCurrent() override; bool GLContextMakeCurrent() override; @@ -47,7 +45,6 @@ class PlatformViewGLFW : public PlatformView, public GPUSurfaceGLDelegate { GLFWwindow* glfw_window_; sky::SkyEnginePtr engine_; int buttons_; - ftl::WeakPtrFactory weak_factory_; void OnWindowSizeChanged(int width, int height); diff --git a/shell/testing/platform_view_test.cc b/shell/testing/platform_view_test.cc index d3e6008a8..04340a36f 100644 --- a/shell/testing/platform_view_test.cc +++ b/shell/testing/platform_view_test.cc @@ -4,21 +4,16 @@ #include "flutter/shell/testing/platform_view_test.h" -#include "flutter/shell/common/shell.h" #include "flutter/shell/common/null_rasterizer.h" +#include "flutter/shell/common/shell.h" namespace shell { PlatformViewTest::PlatformViewTest() - : PlatformView(std::unique_ptr(new NullRasterizer())), - weak_factory_(this) {} + : PlatformView(std::unique_ptr(new NullRasterizer())) {} PlatformViewTest::~PlatformViewTest() = default; -ftl::WeakPtr PlatformViewTest::GetWeakViewPtr() { - return weak_factory_.GetWeakPtr(); -} - bool PlatformViewTest::ResourceContextMakeCurrent() { return false; } diff --git a/shell/testing/platform_view_test.h b/shell/testing/platform_view_test.h index f515ca69a..3d773d9af 100644 --- a/shell/testing/platform_view_test.h +++ b/shell/testing/platform_view_test.h @@ -19,8 +19,6 @@ class PlatformViewTest : public PlatformView { ~PlatformViewTest(); - ftl::WeakPtr GetWeakViewPtr() override; - bool ResourceContextMakeCurrent() override; void RunFromSource(const std::string& main, @@ -28,8 +26,6 @@ class PlatformViewTest : public PlatformView { const std::string& assets_directory) override; private: - ftl::WeakPtrFactory weak_factory_; - FTL_DISALLOW_COPY_AND_ASSIGN(PlatformViewTest); }; -- GitLab