未验证 提交 2592d6ef 编写于 作者: J Jason Simmons 提交者: GitHub

[flutter_runner] Port the accessibility bridge from Topaz (#12054)

上级 b569e8c2
......@@ -914,6 +914,8 @@ FILE: ../../../flutter/shell/platform/fuchsia/dart_runner/service_isolate.cc
FILE: ../../../flutter/shell/platform/fuchsia/dart_runner/service_isolate.h
FILE: ../../../flutter/shell/platform/fuchsia/dart_runner/vmservice/empty.dart
FILE: ../../../flutter/shell/platform/fuchsia/dart_runner/vmservice/meta/vmservice.cmx
FILE: ../../../flutter/shell/platform/fuchsia/flutter/accessibility_bridge.cc
FILE: ../../../flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h
FILE: ../../../flutter/shell/platform/fuchsia/flutter/collect_traces.dart
FILE: ../../../flutter/shell/platform/fuchsia/flutter/component.cc
FILE: ../../../flutter/shell/platform/fuchsia/flutter/component.h
......
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h"
#include <zircon/status.h>
#include <zircon/types.h>
#include <deque>
#include "flutter/fml/logging.h"
#include "flutter/lib/ui/semantics/semantics_node.h"
namespace flutter_runner {
AccessibilityBridge::AccessibilityBridge(
const std::shared_ptr<sys::ServiceDirectory> services,
fuchsia::ui::views::ViewRef view_ref)
: binding_(this) {
services->Connect(fuchsia::accessibility::semantics::SemanticsManager::Name_,
fuchsia_semantics_manager_.NewRequest().TakeChannel());
fuchsia_semantics_manager_.set_error_handler([](zx_status_t status) {
FML_LOG(ERROR) << "Flutter cannot connect to SemanticsManager with status: "
<< zx_status_get_string(status) << ".";
});
fidl::InterfaceHandle<
fuchsia::accessibility::semantics::SemanticActionListener>
listener_handle;
binding_.Bind(listener_handle.NewRequest());
fuchsia_semantics_manager_->RegisterView(
std::move(view_ref), std::move(listener_handle), tree_ptr_.NewRequest());
}
bool AccessibilityBridge::GetSemanticsEnabled() const {
return semantics_enabled_;
}
void AccessibilityBridge::SetSemanticsEnabled(bool enabled) {
semantics_enabled_ = enabled;
if (!enabled) {
nodes_.clear();
}
}
fuchsia::ui::gfx::BoundingBox AccessibilityBridge::GetNodeLocation(
const flutter::SemanticsNode& node) const {
fuchsia::ui::gfx::BoundingBox box;
box.min.x = node.rect.fLeft;
box.min.y = node.rect.fTop;
box.min.z = static_cast<float>(node.elevation);
box.max.x = node.rect.fRight;
box.max.y = node.rect.fBottom;
box.max.z = static_cast<float>(node.thickness);
return box;
}
fuchsia::ui::gfx::mat4 AccessibilityBridge::GetNodeTransform(
const flutter::SemanticsNode& node) const {
fuchsia::ui::gfx::mat4 value;
float* m = value.matrix.data();
node.transform.asColMajorf(m);
return value;
}
fuchsia::accessibility::semantics::Attributes
AccessibilityBridge::GetNodeAttributes(const flutter::SemanticsNode& node,
size_t* added_size) const {
fuchsia::accessibility::semantics::Attributes attributes;
// TODO(MI4-2531): Don't truncate.
if (node.label.size() > fuchsia::accessibility::semantics::MAX_LABEL_SIZE) {
attributes.set_label(node.label.substr(
0, fuchsia::accessibility::semantics::MAX_LABEL_SIZE));
*added_size += fuchsia::accessibility::semantics::MAX_LABEL_SIZE;
} else {
attributes.set_label(node.label);
*added_size += node.label.size();
}
return attributes;
}
fuchsia::accessibility::semantics::States AccessibilityBridge::GetNodeStates(
const flutter::SemanticsNode& node) const {
fuchsia::accessibility::semantics::States states;
if (node.HasFlag(flutter::SemanticsFlags::kHasCheckedState)) {
states.set_checked(node.HasFlag(flutter::SemanticsFlags::kIsChecked));
}
return states;
}
std::unordered_set<int32_t> AccessibilityBridge::GetDescendants(
int32_t node_id) const {
std::unordered_set<int32_t> descendents;
std::deque<int32_t> to_process = {node_id};
while (!to_process.empty()) {
int32_t id = to_process.front();
to_process.pop_front();
descendents.emplace(id);
auto it = nodes_.find(id);
if (it != nodes_.end()) {
auto const& children = it->second;
for (const auto& child : children) {
if (descendents.find(child) == descendents.end()) {
to_process.push_back(child);
} else {
// This indicates either a cycle or a child with multiple parents.
// Flutter should never let this happen, but the engine API does not
// explicitly forbid it right now.
FML_LOG(ERROR) << "Semantics Node " << child
<< " has already been listed as a child of another "
"node, ignoring for parent "
<< id << ".";
}
}
}
}
return descendents;
}
// The only known usage of a negative number for a node ID is in the embedder
// API as a sentinel value, which is not expected here. No valid producer of
// nodes should give us a negative ID.
static uint32_t FlutterIdToFuchsiaId(int32_t flutter_node_id) {
FML_DCHECK(flutter_node_id >= 0)
<< "Unexpectedly recieved a negative semantics node ID.";
return static_cast<uint32_t>(flutter_node_id);
}
void AccessibilityBridge::PruneUnreachableNodes() {
const auto& reachable_nodes = GetDescendants(kRootNodeId);
std::vector<uint32_t> nodes_to_remove;
auto iter = nodes_.begin();
while (iter != nodes_.end()) {
int32_t id = iter->first;
if (reachable_nodes.find(id) == reachable_nodes.end()) {
// TODO(MI4-2531): This shouldn't be strictly necessary at this level.
if (sizeof(nodes_to_remove) + (nodes_to_remove.size() * kNodeIdSize) >=
kMaxMessageSize) {
tree_ptr_->DeleteSemanticNodes(std::move(nodes_to_remove));
nodes_to_remove.clear();
}
nodes_to_remove.push_back(FlutterIdToFuchsiaId(id));
iter = nodes_.erase(iter);
} else {
iter++;
}
}
if (!nodes_to_remove.empty()) {
tree_ptr_->DeleteSemanticNodes(std::move(nodes_to_remove));
}
}
// TODO(FIDL-718) - remove this, handle the error instead in something like
// set_error_handler.
static void PrintNodeSizeError(uint32_t node_id) {
FML_LOG(ERROR) << "Semantics node with ID " << node_id
<< " exceeded the maximum FIDL message size and may not "
"be delivered to the accessibility manager service.";
}
void AccessibilityBridge::AddSemanticsNodeUpdate(
const flutter::SemanticsNodeUpdates update) {
if (update.empty()) {
return;
}
FML_DCHECK(nodes_.find(kRootNodeId) != nodes_.end() ||
update.find(kRootNodeId) != update.end())
<< "AccessibilityBridge received an update with out ever getting a root "
"node.";
std::vector<fuchsia::accessibility::semantics::Node> nodes;
size_t current_size = 0;
// TODO(MI4-2498): Actions, Roles, hit test children, additional
// flags/states/attr
// TODO(MI4-1478): Support for partial updates for nodes > 64kb
// e.g. if a node has a long label or more than 64k children.
for (const auto& value : update) {
size_t this_node_size = sizeof(fuchsia::accessibility::semantics::Node);
const auto& flutter_node = value.second;
nodes_[flutter_node.id] =
std::vector<int32_t>(flutter_node.childrenInTraversalOrder);
fuchsia::accessibility::semantics::Node fuchsia_node;
std::vector<uint32_t> child_ids;
for (int32_t flutter_child_id : flutter_node.childrenInTraversalOrder) {
child_ids.push_back(FlutterIdToFuchsiaId(flutter_child_id));
}
fuchsia_node.set_node_id(flutter_node.id)
.set_location(GetNodeLocation(flutter_node))
.set_transform(GetNodeTransform(flutter_node))
.set_attributes(GetNodeAttributes(flutter_node, &this_node_size))
.set_states(GetNodeStates(flutter_node))
.set_child_ids(child_ids);
this_node_size +=
kNodeIdSize * flutter_node.childrenInTraversalOrder.size();
// TODO(MI4-2531, FIDL-718): Remove this
// This is defensive. If, despite our best efforts, we ended up with a node
// that is larger than the max fidl size, we send no updates.
if (this_node_size >= kMaxMessageSize) {
PrintNodeSizeError(flutter_node.id);
return;
}
current_size += this_node_size;
// If we would exceed the max FIDL message size by appending this node,
// we should delete/update/commit now.
if (current_size >= kMaxMessageSize) {
tree_ptr_->UpdateSemanticNodes(std::move(nodes));
nodes.clear();
current_size = this_node_size;
}
nodes.push_back(std::move(fuchsia_node));
}
if (current_size > kMaxMessageSize) {
PrintNodeSizeError(nodes.back().node_id());
}
PruneUnreachableNodes();
tree_ptr_->UpdateSemanticNodes(std::move(nodes));
tree_ptr_->Commit();
}
// |fuchsia::accessibility::semantics::SemanticActionListener|
void AccessibilityBridge::OnAccessibilityActionRequested(
uint32_t node_id,
fuchsia::accessibility::semantics::Action action,
fuchsia::accessibility::semantics::SemanticActionListener::
OnAccessibilityActionRequestedCallback callback) {}
// |fuchsia::accessibility::semantics::SemanticActionListener|
void AccessibilityBridge::HitTest(
fuchsia::math::PointF local_point,
fuchsia::accessibility::semantics::SemanticActionListener::HitTestCallback
callback) {}
} // namespace flutter_runner
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef TOPAZ_RUNTIME_FLUTTER_RUNNER_ACCESSIBILITY_BRIDGE_H_
#define TOPAZ_RUNTIME_FLUTTER_RUNNER_ACCESSIBILITY_BRIDGE_H_
#include <fuchsia/accessibility/semantics/cpp/fidl.h>
#include <fuchsia/sys/cpp/fidl.h>
#include <fuchsia/ui/gfx/cpp/fidl.h>
#include <lib/fidl/cpp/binding_set.h>
#include <lib/sys/cpp/service_directory.h>
#include <zircon/types.h>
#include <memory>
#include <optional>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "flutter/fml/macros.h"
#include "flutter/lib/ui/semantics/semantics_node.h"
namespace flutter_runner {
// Accessibility bridge.
//
// This class intermediates accessibility-related calls between Fuchsia and
// Flutter. It serves to resolve the impedance mismatch between Flutter's
// platform-agnostic accessibility APIs and Fuchsia's APIs and behaviour.
//
// This bridge performs the following functions, among others:
//
// * Translates Flutter's semantics node updates to events Fuchsia requires
// (e.g. Flutter only sends updates for changed nodes, but Fuchsia requires
// the entire flattened subtree to be sent when a node changes.
class AccessibilityBridge
: public fuchsia::accessibility::semantics::SemanticActionListener {
public:
// TODO(MI4-2531, FIDL-718): Remove this. We shouldn't be worried about
// batching messages at this level.
// FIDL may encode a C++ struct as larger than the sizeof the C++ struct.
// This is to make sure we don't send updates that are too large.
static constexpr uint32_t kMaxMessageSize = ZX_CHANNEL_MAX_MSG_BYTES / 2;
static_assert(fuchsia::accessibility::semantics::MAX_LABEL_SIZE <
kMaxMessageSize - 1);
// Flutter uses signed 32 bit integers for node IDs, while Fuchsia uses
// unsigned 32 bit integers. A change in the size on either one would break
// casts and size tracking logic in the implementation.
static constexpr size_t kNodeIdSize = sizeof(flutter::SemanticsNode::id);
static_assert(
kNodeIdSize ==
sizeof(fuchsia::accessibility::semantics::Node().node_id()),
"flutter::SemanticsNode::id and "
"fuchsia::accessibility::semantics::Node::node_id differ in size.");
AccessibilityBridge(const std::shared_ptr<sys::ServiceDirectory> services,
fuchsia::ui::views::ViewRef view_ref);
// Returns true if accessible navigation is enabled.
bool GetSemanticsEnabled() const;
// Enables Flutter accessibility navigation features.
//
// Once enabled, any semantics updates in the Flutter application will
// trigger |FuchsiaAccessibility::DispatchAccessibilityEvent| callbacks
// to send events back to the Fuchsia SemanticsManager.
void SetSemanticsEnabled(bool enabled);
// Adds a semantics node update to the buffer of node updates to apply.
void AddSemanticsNodeUpdate(const flutter::SemanticsNodeUpdates update);
// Notifies the bridge of a 'hover move' touch exploration event.
zx_status_t OnHoverMove(double x, double y);
private:
static constexpr int32_t kRootNodeId = 0;
fidl::Binding<fuchsia::accessibility::semantics::SemanticActionListener>
binding_;
fuchsia::accessibility::semantics::SemanticsManagerPtr
fuchsia_semantics_manager_;
fuchsia::accessibility::semantics::SemanticTreePtr tree_ptr_;
bool semantics_enabled_;
// This is the cache of all nodes we've sent to Fuchsia's SemanticsManager.
// Assists with pruning unreachable nodes.
std::unordered_map<int32_t, std::vector<int32_t>> nodes_;
// Derives the BoundingBox of a Flutter semantics node from its
// rect and elevation.
fuchsia::ui::gfx::BoundingBox GetNodeLocation(
const flutter::SemanticsNode& node) const;
// Converts a Flutter semantics node's transformation to a mat4.
fuchsia::ui::gfx::mat4 GetNodeTransform(
const flutter::SemanticsNode& node) const;
// Derives the attributes for a Fuchsia semantics node from a Flutter
// semantics node.
fuchsia::accessibility::semantics::Attributes GetNodeAttributes(
const flutter::SemanticsNode& node,
size_t* added_size) const;
// Derives the states for a Fuchsia semantics node from a Flutter semantics
// node.
fuchsia::accessibility::semantics::States GetNodeStates(
const flutter::SemanticsNode& node) const;
// Gets the set of reachable descendants from the given node id.
std::unordered_set<int32_t> GetDescendants(int32_t node_id) const;
// Removes internal references to any dangling nodes from previous
// updates, and updates the Accessibility service.
//
// May result in a call to FuchsiaAccessibility::Commit().
void PruneUnreachableNodes();
// |fuchsia::accessibility::semantics::SemanticActionListener|
void OnAccessibilityActionRequested(
uint32_t node_id,
fuchsia::accessibility::semantics::Action action,
fuchsia::accessibility::semantics::SemanticActionListener::
OnAccessibilityActionRequestedCallback callback) override;
// |fuchsia::accessibility::semantics::SemanticActionListener|
void HitTest(
fuchsia::math::PointF local_point,
fuchsia::accessibility::semantics::SemanticActionListener::HitTestCallback
callback) override;
FML_DISALLOW_COPY_AND_ASSIGN(AccessibilityBridge);
};
} // namespace flutter_runner
#endif // TOPAZ_RUNTIME_FLUTTER_RUNNER_ACCESSIBILITY_BRIDGE_H_
......@@ -526,14 +526,28 @@ void Application::CreateView(
return;
}
// TODO(MI4-2490): remove once ViewRefControl and ViewRef come as a parameters
// to CreateView
fuchsia::ui::views::ViewRefControl view_ref_control;
fuchsia::ui::views::ViewRef view_ref;
zx_status_t status = zx::eventpair::create(
/*flags*/ 0u, &view_ref_control.reference, &view_ref.reference);
FML_DCHECK(status == ZX_OK);
status = view_ref.reference.replace(ZX_RIGHTS_BASIC, &view_ref.reference);
FML_DCHECK(status == ZX_OK);
shell_holders_.emplace(std::make_unique<Engine>(
*this, // delegate
debug_label_, // thread label
svc_, // Component incoming services
runner_incoming_services_, // Runner incoming services
settings_, // settings
std::move(isolate_snapshot_), // isolate snapshot
std::move(shared_snapshot_), // shared snapshot
scenic::ToViewToken(std::move(view_token)), // view token
std::move(view_ref_control), // view ref control
std::move(view_ref), // view ref
std::move(fdio_ns_), // FDIO namespace
std::move(directory_request_) // outgoing request
));
......
......@@ -42,10 +42,13 @@ static void UpdateNativeThreadLabelNames(const std::string& label,
Engine::Engine(Delegate& delegate,
std::string thread_label,
std::shared_ptr<sys::ServiceDirectory> svc,
std::shared_ptr<sys::ServiceDirectory> runner_services,
flutter::Settings settings,
fml::RefPtr<const flutter::DartSnapshot> isolate_snapshot,
fml::RefPtr<const flutter::DartSnapshot> shared_snapshot,
fuchsia::ui::views::ViewToken view_token,
fuchsia::ui::views::ViewRefControl view_ref_control,
fuchsia::ui::views::ViewRef view_ref,
UniqueFDIONS fdio_ns,
fidl::InterfaceRequest<fuchsia::io::Directory> directory_request)
: delegate_(delegate),
......@@ -107,6 +110,9 @@ Engine::Engine(Delegate& delegate,
flutter::Shell::CreateCallback<flutter::PlatformView>
on_create_platform_view = fml::MakeCopyable(
[debug_label = thread_label_,
view_ref_control = std::move(view_ref_control),
view_ref = std::move(view_ref),
runner_services = std::move(runner_services),
parent_environment_service_provider =
std::move(parent_environment_service_provider),
session_listener_request = std::move(session_listener_request),
......@@ -120,9 +126,12 @@ Engine::Engine(Delegate& delegate,
std::move(on_enable_wireframe_callback),
vsync_handle = vsync_event_.get()](flutter::Shell& shell) mutable {
return std::make_unique<flutter_runner::PlatformView>(
shell, // delegate
debug_label, // debug label
shell.GetTaskRunners(), // task runners
shell, // delegate
debug_label, // debug label
std::move(view_ref_control), // view control ref
std::move(view_ref), // view ref
shell.GetTaskRunners(), // task runners
std::move(runner_services),
std::move(parent_environment_service_provider), // services
std::move(session_listener_request), // session listener
std::move(on_session_listener_error_callback),
......
......@@ -31,10 +31,13 @@ class Engine final {
Engine(Delegate& delegate,
std::string thread_label,
std::shared_ptr<sys::ServiceDirectory> svc,
std::shared_ptr<sys::ServiceDirectory> runner_services,
flutter::Settings settings,
fml::RefPtr<const flutter::DartSnapshot> isolate_snapshot,
fml::RefPtr<const flutter::DartSnapshot> shared_snapshot,
fuchsia::ui::views::ViewToken view_token,
fuchsia::ui::views::ViewRefControl view_ref_control,
fuchsia::ui::views::ViewRef view_ref,
UniqueFDIONS fdio_ns,
fidl::InterfaceRequest<fuchsia::io::Directory> directory_request);
~Engine();
......
......@@ -42,6 +42,8 @@ template("flutter_runner") {
libs = []
sources = [
"accessibility_bridge.cc",
"accessibility_bridge.h",
"component.cc",
"component.h",
"compositor_context.cc",
......@@ -110,6 +112,7 @@ template("flutter_runner") {
]
deps = [
"$fuchsia_sdk_root/fidl:fuchsia.accessibility.semantics",
"$fuchsia_sdk_root/fidl:fuchsia.fonts",
"$fuchsia_sdk_root/fidl:fuchsia.images",
"$fuchsia_sdk_root/fidl:fuchsia.io",
......
......@@ -79,7 +79,10 @@ void SetInterfaceErrorHandler(fidl::Binding<T>& binding, std::string name) {
PlatformView::PlatformView(
PlatformView::Delegate& delegate,
std::string debug_label,
fuchsia::ui::views::ViewRefControl view_ref_control,
fuchsia::ui::views::ViewRef view_ref,
flutter::TaskRunners task_runners,
std::shared_ptr<sys::ServiceDirectory> runner_services,
fidl::InterfaceHandle<fuchsia::sys::ServiceProvider>
parent_environment_service_provider_handle,
fidl::InterfaceRequest<fuchsia::ui::scenic::SessionListener>
......@@ -91,6 +94,8 @@ PlatformView::PlatformView(
zx_handle_t vsync_event_handle)
: flutter::PlatformView(delegate, std::move(task_runners)),
debug_label_(std::move(debug_label)),
view_ref_control_(std::move(view_ref_control)),
view_ref_(std::move(view_ref)),
session_listener_binding_(this, std::move(session_listener_request)),
session_listener_error_callback_(
std::move(session_listener_error_callback)),
......@@ -121,6 +126,11 @@ PlatformView::PlatformView(
// Finally! Register the native platform message handlers.
RegisterPlatformMessageHandlers();
fuchsia::ui::views::ViewRef accessibility_view_ref;
view_ref_.Clone(&accessibility_view_ref);
accessibility_bridge_ = std::make_unique<AccessibilityBridge>(
runner_services, std::move(accessibility_view_ref));
// TODO(SCN-975): Re-enable. Likely that Engine should clone the ViewToken
// and pass the clone in here.
// view_->GetToken(std::bind(&PlatformView::ConnectSemanticsProvider, this,
......@@ -590,12 +600,17 @@ void PlatformView::HandlePlatformMessage(
found->second(std::move(message));
}
// |flutter::PlatformView|
void PlatformView::SetSemanticsEnabled(bool enabled) {
accessibility_bridge_->SetSemanticsEnabled(enabled);
flutter::PlatformView::SetSemanticsEnabled(enabled);
}
// |flutter::PlatformView|
void PlatformView::UpdateSemantics(
flutter::SemanticsNodeUpdates update,
flutter::CustomAccessibilityActionUpdates actions) {
// TODO(MIT-1539): Uncomment/Reimplement following code, to add A11y support.
// semantics_bridge_.UpdateSemantics(update);
accessibility_bridge_->AddSemanticsNodeUpdate(update);
}
// Channel handler for kAccessibilityChannel
......
......@@ -17,6 +17,7 @@
#include "flutter/fml/macros.h"
#include "flutter/lib/ui/window/viewport_metrics.h"
#include "flutter/shell/common/platform_view.h"
#include "flutter/shell/platform/fuchsia/flutter/accessibility_bridge.h"
#include "lib/fidl/cpp/binding.h"
#include "lib/ui/scenic/cpp/id.h"
......@@ -41,7 +42,10 @@ class PlatformView final : public flutter::PlatformView,
public:
PlatformView(PlatformView::Delegate& delegate,
std::string debug_label,
fuchsia::ui::views::ViewRefControl view_ref_control,
fuchsia::ui::views::ViewRef view_ref,
flutter::TaskRunners task_runners,
std::shared_ptr<sys::ServiceDirectory> runner_services,
fidl::InterfaceHandle<fuchsia::sys::ServiceProvider>
parent_environment_service_provider,
fidl::InterfaceRequest<fuchsia::ui::scenic::SessionListener>
......@@ -64,6 +68,11 @@ class PlatformView final : public flutter::PlatformView,
private:
const std::string debug_label_;
// TODO(MI4-2490): remove once ViewRefControl is passed to Scenic and kept
// alive there
const fuchsia::ui::views::ViewRefControl view_ref_control_;
const fuchsia::ui::views::ViewRef view_ref_;
std::unique_ptr<AccessibilityBridge> accessibility_bridge_;
fidl::Binding<fuchsia::ui::scenic::SessionListener> session_listener_binding_;
fit::closure session_listener_error_callback_;
......@@ -144,6 +153,9 @@ class PlatformView final : public flutter::PlatformView,
void HandlePlatformMessage(
fml::RefPtr<flutter::PlatformMessage> message) override;
// |flutter::PlatformView|
void SetSemanticsEnabled(bool enabled) override;
// |flutter::PlatformView|
void UpdateSemantics(
flutter::SemanticsNodeUpdates update,
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册