From abe9826a9d328731ae96bc913591cd10f2e53883 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Wed, 20 Feb 2019 18:59:29 -0800 Subject: [PATCH] Add accessibility semantics support to embedder (#7891) Flutter's accessibility APIs consist of three main calls from the embedder to the Dart application: 1. FlutterEngineUpdateSemanticsEnabled: enables/disables semantics support. 2. FlutterEngineUpdateAccessibilityFeatures: sets embedder-specific accessibility features. 3. FlutterEngineDispatchSemanticsAction: dispatches an action (tap, long-press, scroll, etc.) to a semantics node. and two main callbacks triggered by Dart code: 1. FlutterUpdateSemanticsNodeCallback: notifies the embedder of updates to the properties of a given semantics node. 2. FlutterUpdateSemanticsCustomActionCallback: notifies the embedder of updates to custom semantics actions registered in Dart code. In the Flutter framework, when accessibility is first enabled, the embedder will receive a stream of update callbacks notifying the embedder of the full semantics tree. On further changes in the Dart application, only updates will be sent. --- BUILD.gn | 1 + ci/licenses_golden/licenses_flutter | 1 + lib/ui/semantics.dart | 6 + lib/ui/window.dart | 3 + shell/platform/embedder/BUILD.gn | 28 ++ shell/platform/embedder/embedder.cc | 113 ++++++++ shell/platform/embedder/embedder.h | 261 ++++++++++++++++++ shell/platform/embedder/embedder_engine.cc | 45 +++ shell/platform/embedder/embedder_engine.h | 9 + .../platform/embedder/fixtures/a11y_main.dart | 112 ++++++++ .../embedder/platform_view_embedder.cc | 13 + .../embedder/platform_view_embedder.h | 12 + .../embedder/tests/embedder_a11y_unittests.cc | 205 ++++++++++++++ 13 files changed, 809 insertions(+) create mode 100644 shell/platform/embedder/fixtures/a11y_main.dart create mode 100644 shell/platform/embedder/tests/embedder_a11y_unittests.cc diff --git a/BUILD.gn b/BUILD.gn index cc9b9dff1..55f78f785 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -57,6 +57,7 @@ group("flutter") { "$flutter_root/runtime:runtime_unittests", "$flutter_root/shell/common:shell_unittests", "$flutter_root/shell/platform/embedder:embedder_unittests", + "$flutter_root/shell/platform/embedder:embedder_a11y_unittests", # TODO(cbracken) build these into a different kernel blob in the embedder tests and load that in a test in embedder_unittests "$flutter_root/synchronization:synchronization_unittests", "$flutter_root/third_party/txt:txt_unittests", ] diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 4b4db2bad..89ccd037e 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -619,6 +619,7 @@ FILE: ../../../flutter/shell/platform/embedder/embedder_surface_gl.cc FILE: ../../../flutter/shell/platform/embedder/embedder_surface_gl.h FILE: ../../../flutter/shell/platform/embedder/embedder_surface_software.cc FILE: ../../../flutter/shell/platform/embedder/embedder_surface_software.h +FILE: ../../../flutter/shell/platform/embedder/fixtures/a11y_main.dart FILE: ../../../flutter/shell/platform/embedder/fixtures/simple_main.dart FILE: ../../../flutter/shell/platform/embedder/platform_view_embedder.cc FILE: ../../../flutter/shell/platform/embedder/platform_view_embedder.h diff --git a/lib/ui/semantics.dart b/lib/ui/semantics.dart index 750f7f017..5d54eabf6 100644 --- a/lib/ui/semantics.dart +++ b/lib/ui/semantics.dart @@ -6,6 +6,9 @@ part of dart.ui; /// The possible actions that can be conveyed from the operating system /// accessibility APIs to a semantics node. +// +// When changes are made to this class, the equivalent APIs in each of the +// embedders *must* be updated. class SemanticsAction { const SemanticsAction._(this.index); @@ -260,6 +263,9 @@ class SemanticsAction { } /// A Boolean value that can be associated with a semantics node. +// +// When changes are made to this class, the equivalent APIs in each of the +// embedders *must* be updated. class SemanticsFlag { static const int _kHasCheckedStateIndex = 1 << 0; static const int _kIsCheckedIndex = 1 << 1; diff --git a/lib/ui/window.dart b/lib/ui/window.dart index c33ed641e..56b346c33 100644 --- a/lib/ui/window.dart +++ b/lib/ui/window.dart @@ -883,6 +883,9 @@ class Window { /// It is not possible to enable these settings from Flutter, instead they are /// used by the platform to indicate that additional accessibility features are /// enabled. +// +// When changes are made to this class, the equivalent APIs in each of the +// embedders *must* be updated. class AccessibilityFeatures { const AccessibilityFeatures._(this._index); diff --git a/shell/platform/embedder/BUILD.gn b/shell/platform/embedder/BUILD.gn index 0d4aedb0d..2faf89eef 100644 --- a/shell/platform/embedder/BUILD.gn +++ b/shell/platform/embedder/BUILD.gn @@ -80,6 +80,34 @@ executable("embedder_unittests") { } } +test_fixtures("fixtures_a11y") { + fixtures = [ "fixtures/a11y_main.dart" ] +} + +executable("embedder_a11y_unittests") { + testonly = true + + include_dirs = [ "." ] + + sources = [ + "tests/embedder_a11y_unittests.cc", + ] + + deps = [ + ":embedder", + ":fixtures_a11y", + "$flutter_root/lib/ui:ui", + "$flutter_root/shell/common", + "$flutter_root/testing", + "//third_party/skia", + "//third_party/tonic", + ] + + if (is_linux) { + ldflags = [ "-rdynamic" ] + } +} + shared_library("flutter_engine_library") { visibility = [ ":*" ] diff --git a/shell/platform/embedder/embedder.cc b/shell/platform/embedder/embedder.cc index d134847ed..f0f90c125 100644 --- a/shell/platform/embedder/embedder.cc +++ b/shell/platform/embedder/embedder.cc @@ -383,6 +383,74 @@ FlutterEngineResult FlutterEngineRun(size_t version, thread_host.io_thread->GetTaskRunner() // io ); + shell::PlatformViewEmbedder::UpdateSemanticsNodesCallback + update_semantics_nodes_callback = nullptr; + if (SAFE_ACCESS(args, update_semantics_node_callback, nullptr) != nullptr) { + update_semantics_nodes_callback = + [ptr = args->update_semantics_node_callback, + user_data](blink::SemanticsNodeUpdates update) { + for (const auto& value : update) { + const auto& node = value.second; + const auto& transform = node.transform; + auto flutter_transform = FlutterTransformation{ + transform.get(0, 0), transform.get(0, 1), transform.get(0, 2), + transform.get(1, 0), transform.get(1, 1), transform.get(1, 2), + transform.get(2, 0), transform.get(2, 1), transform.get(2, 2)}; + const FlutterSemanticsNode embedder_node = { + sizeof(FlutterSemanticsNode), + node.id, + static_cast(node.flags), + static_cast(node.actions), + node.textSelectionBase, + node.textSelectionExtent, + node.scrollChildren, + node.scrollIndex, + node.scrollPosition, + node.scrollExtentMax, + node.scrollExtentMin, + node.elevation, + node.thickness, + node.label.c_str(), + node.hint.c_str(), + node.value.c_str(), + node.increasedValue.c_str(), + node.decreasedValue.c_str(), + static_cast(node.textDirection), + FlutterRect{node.rect.fLeft, node.rect.fTop, node.rect.fRight, + node.rect.fBottom}, + flutter_transform, + node.childrenInTraversalOrder.size(), + &node.childrenInTraversalOrder[0], + &node.childrenInHitTestOrder[0], + node.customAccessibilityActions.size(), + &node.customAccessibilityActions[0], + }; + ptr(&embedder_node, user_data); + } + }; + } + + shell::PlatformViewEmbedder::UpdateSemanticsCustomActionsCallback + update_semantics_custom_actions_callback = nullptr; + if (SAFE_ACCESS(args, update_semantics_custom_action_callback, nullptr) != + nullptr) { + update_semantics_custom_actions_callback = + [ptr = args->update_semantics_custom_action_callback, + user_data](blink::CustomAccessibilityActionUpdates actions) { + for (const auto& value : actions) { + const auto& action = value.second; + const FlutterSemanticsCustomAction embedder_action = { + sizeof(FlutterSemanticsCustomAction), + action.id, + static_cast(action.overrideId), + action.label.c_str(), + action.hint.c_str(), + }; + ptr(&embedder_action, user_data); + } + }; + } + shell::PlatformViewEmbedder::PlatformMessageResponseCallback platform_message_response_callback = nullptr; if (SAFE_ACCESS(args, platform_message_callback, nullptr) != nullptr) { @@ -403,6 +471,7 @@ FlutterEngineResult FlutterEngineRun(size_t version, } shell::PlatformViewEmbedder::PlatformDispatchTable platform_dispatch_table = { + update_semantics_nodes_callback, update_semantics_custom_actions_callback, platform_message_response_callback, // platform_message_response_callback }; @@ -688,3 +757,47 @@ FlutterEngineResult FlutterEngineMarkExternalTextureFrameAvailable( } return kSuccess; } + +FlutterEngineResult FlutterEngineUpdateSemanticsEnabled(FlutterEngine engine, + bool enabled) { + if (engine == nullptr) { + return kInvalidArguments; + } + if (!reinterpret_cast(engine)->SetSemanticsEnabled( + enabled)) { + return kInternalInconsistency; + } + return kSuccess; +} + +FlutterEngineResult FlutterEngineUpdateAccessibilityFeatures( + FlutterEngine engine, + FlutterAccessibilityFeature flags) { + if (engine == nullptr) { + return kInvalidArguments; + } + if (!reinterpret_cast(engine) + ->SetAccessibilityFeatures(flags)) { + return kInternalInconsistency; + } + return kSuccess; +} + +FlutterEngineResult FlutterEngineDispatchSemanticsAction( + FlutterEngine engine, + uint64_t id, + FlutterSemanticsAction action, + const uint8_t* data, + size_t data_length) { + if (engine == nullptr) { + return kInvalidArguments; + } + auto engine_action = static_cast(action); + if (!reinterpret_cast(engine) + ->DispatchSemanticsAction( + id, engine_action, + std::vector({data, data + data_length}))) { + return kInternalInconsistency; + } + return kSuccess; +} diff --git a/shell/platform/embedder/embedder.h b/shell/platform/embedder/embedder.h index 010bfbe33..15c5e4922 100644 --- a/shell/platform/embedder/embedder.h +++ b/shell/platform/embedder/embedder.h @@ -31,6 +31,127 @@ typedef enum { kSoftware, } FlutterRendererType; +// Additional accessibility features that may be enabled by the platform. +// +// Must match the |AccessibilityFeatures| enum in window.dart. +typedef enum { + // Indicate there is a running accessibility service which is changing the + // interaction model of the device. + kFlutterAccessibilityFeatureAccessibleNavigation = 1 << 0, + // Indicate the platform is inverting the colors of the application. + kFlutterAccessibilityFeatureInvertColors = 1 << 1, + // Request that animations be disabled or simplified. + kFlutterAccessibilityFeatureDisableAnimations = 1 << 2, + // Request that text be rendered at a bold font weight. + kFlutterAccessibilityFeatureBoldText = 1 << 3, + // Request that certain animations be simplified and parallax effects + // removed. + kFlutterAccessibilityFeatureReduceMotion = 1 << 4, +} FlutterAccessibilityFeature; + +// The set of possible actions that can be conveyed to a semantics node. +// +// Must match the |SemanticsAction| enum in semantics.dart. +typedef enum { + // The equivalent of a user briefly tapping the screen with the finger without + // moving it. + kFlutterSemanticsActionTap = 1 << 0, + // The equivalent of a user pressing and holding the screen with the finger + // for a few seconds without moving it. + kFlutterSemanticsActionLongPress = 1 << 1, + // The equivalent of a user moving their finger across the screen from right + // to left. + kFlutterSemanticsActionScrollLeft = 1 << 2, + // The equivalent of a user moving their finger across the screen from left to + // right. + kFlutterSemanticsActionScrollRight = 1 << 3, + // The equivalent of a user moving their finger across the screen from bottom + // to top. + kFlutterSemanticsActionScrollUp = 1 << 4, + // The equivalent of a user moving their finger across the screen from top to + // bottom. + kFlutterSemanticsActionScrollDown = 1 << 5, + // Increase the value represented by the semantics node. + kFlutterSemanticsActionIncrease = 1 << 6, + // Decrease the value represented by the semantics node. + kFlutterSemanticsActionDecrease = 1 << 7, + // A request to fully show the semantics node on screen. + kFlutterSemanticsActionShowOnScreen = 1 << 8, + // Move the cursor forward by one character. + kFlutterSemanticsActionMoveCursorForwardByCharacter = 1 << 9, + // Move the cursor backward by one character. + kFlutterSemanticsActionMoveCursorBackwardByCharacter = 1 << 10, + // Set the text selection to the given range. + kFlutterSemanticsActionSetSelection = 1 << 11, + // Copy the current selection to the clipboard. + kFlutterSemanticsActionCopy = 1 << 12, + // Cut the current selection and place it in the clipboard. + kFlutterSemanticsActionCut = 1 << 13, + // Paste the current content of the clipboard. + kFlutterSemanticsActionPaste = 1 << 14, + // Indicate that the node has gained accessibility focus. + kFlutterSemanticsActionDidGainAccessibilityFocus = 1 << 15, + // Indicate that the node has lost accessibility focus. + kFlutterSemanticsActionDidLoseAccessibilityFocus = 1 << 16, + // Indicate that the user has invoked a custom accessibility action. + kFlutterSemanticsActionCustomAction = 1 << 17, + // A request that the node should be dismissed. + kFlutterSemanticsActionDismiss = 1 << 18, +} FlutterSemanticsAction; + +// The set of properties that may be associated with a semantics node. +// +// Must match the |SemanticsFlag| enum in semantics.dart. +typedef enum { + // The semantics node has the quality of either being "checked" or + // "unchecked". + kFlutterSemanticsFlagHasCheckedState = 1 << 0, + // Whether a semantics node is checked. + kFlutterSemanticsFlagIsChecked = 1 << 1, + // Whether a semantics node is selected. + kFlutterSemanticsFlagIsSelected = 1 << 2, + // Whether the semantic node represents a button. + kFlutterSemanticsFlagIsButton = 1 << 3, + // Whether the semantic node represents a text field. + kFlutterSemanticsFlagIsTextField = 1 << 4, + // Whether the semantic node currently holds the user's focus. + kFlutterSemanticsFlagIsFocused = 1 << 5, + // The semantics node has the quality of either being "enabled" or "disabled". + kFlutterSemanticsFlagHasEnabledState = 1 << 6, + // Whether a semantic node that hasEnabledState is currently enabled. + kFlutterSemanticsFlagIsEnabled = 1 << 7, + // Whether a semantic node is in a mutually exclusive group. + kFlutterSemanticsFlagIsInMutuallyExclusiveGroup = 1 << 8, + // Whether a semantic node is a header that divides content into sections. + kFlutterSemanticsFlagIsHeader = 1 << 9, + // Whether the value of the semantics node is obscured. + kFlutterSemanticsFlagIsObscured = 1 << 10, + // Whether the semantics node is the root of a subtree for which a route name + // should be announced. + kFlutterSemanticsFlagScopesRoute = 1 << 11, + // Whether the semantics node label is the name of a visually distinct route. + kFlutterSemanticsFlagNamesRoute = 1 << 12, + // Whether the semantics node is considered hidden. + kFlutterSemanticsFlagIsHidden = 1 << 13, + // Whether the semantics node represents an image. + kFlutterSemanticsFlagIsImage = 1 << 14, + // Whether the semantics node is a live region. + kFlutterSemanticsFlagIsLiveRegion = 1 << 15, + // The semantics node has the quality of either being "on" or "off". + kFlutterSemanticsFlagHasToggledState = 1 << 16, + // If true, the semantics node is "on". If false, the semantics node is "off". + kFlutterSemanticsFlagIsToggled = 1 << 17, +} FlutterSemanticsFlag; + +typedef enum { + // Text has unknown text direction. + kFlutterTextDirectionUnknown = 0, + // Text is read from right to left. + kFlutterTextDirectionRTL = 1, + // Text is read from left to right. + kFlutterTextDirectionLTR = 2, +} FlutterTextDirection; + typedef struct _FlutterEngine* FlutterEngine; typedef struct { @@ -190,6 +311,111 @@ typedef void (*FlutterPlatformMessageCallback)( const FlutterPlatformMessage* /* message*/, void* /* user data */); +typedef struct { + double left; + double top; + double right; + double bottom; +} FlutterRect; + +// A node that represents some semantic data. +// +// The semantics tree is maintained during the semantics phase of the pipeline +// (i.e., during PipelineOwner.flushSemantics), which happens after +// compositing. Updates are then pushed to embedders via the registered +// |FlutterUpdateSemanticsNodeCallback|. +typedef struct { + // The size of this struct. Must be sizeof(FlutterSemanticsNode). + size_t struct_size; + // The unique identifier for this node. + int32_t id; + // The set of semantics flags associated with this node. + FlutterSemanticsFlag flags; + // The set of semantics actions applicable to this node. + FlutterSemanticsAction actions; + // The position at which the text selection originates. + int32_t textSelectionBase; + // The position at which the text selection terminates. + int32_t textSelectionExtent; + // The total number of scrollable children that contribute to semantics. + int32_t scrollChildren; + // The index of the first visible semantic child of a scroll node. + int32_t scrollIndex; + // The current scrolling position in logical pixels if the node is scrollable. + double scrollPosition; + // The maximum in-range value for |scrollPosition| if the node is scrollable. + double scrollExtentMax; + // The minimum in-range value for |scrollPosition| if the node is scrollable. + double scrollExtentMin; + // The elevation along the z-axis at which the rect of this semantics node is + // located above its parent. + double elevation; + // Describes how much space the semantics node takes up along the z-axis. + double thickness; + // A textual description of the node. + const char* label; + // A brief description of the result of performing an action on the node. + const char* hint; + // A textual description of the current value of the node. + const char* value; + // A value that |value| will have after a kFlutterSemanticsActionIncrease| + // action has been performed. + const char* increasedValue; + // A value that |value| will have after a kFlutterSemanticsActionDecrease| + // action has been performed. + const char* decreasedValue; + // The reading direction for |label|, |value|, |hint|, |increasedValue|, and + // |decreasedValue|. + FlutterTextDirection textDirection; + // The bounding box for this node in its coordinate system. + FlutterRect rect; + // The transform from this node's coordinate system to its parent's coordinate + // system. + FlutterTransformation transform; + // The number of children this node has. + size_t child_count; + // Array of child node IDs in traversal order. Has length |child_count|. + const int32_t* children_in_traversal_order; + // Array of child node IDs in hit test order. Has length |child_count|. + const int32_t* children_in_hit_test_order; + // The number of custom accessibility action associated with this node. + size_t custom_accessibility_actions_count; + // Array of |FlutterSemanticsCustomAction| IDs associated with this node. + // Has length |custom_accessibility_actions_count|. + const int32_t* custom_accessibility_actions; +} FlutterSemanticsNode; + +// A custom semantics action, or action override. +// +// Custom actions can be registered by applications in order to provide +// semantic actions other than the standard actions available through the +// |FlutterSemanticsAction| enum. +// +// Action overrides are custom actions that the application developer requests +// to be used in place of the standard actions in the |FlutterSemanticsAction| +// enum. +typedef struct { + // The size of the struct. Must be sizeof(FlutterSemanticsCustomAction). + size_t struct_size; + // The unique custom action or action override ID. + int32_t id; + // For overriden standard actions, corresponds to the + // |FlutterSemanticsAction| to override. + FlutterSemanticsAction override_action; + // The user-readable name of this custom semantics action. + const char* label; + // The hint description of this custom semantics action. + const char* hint; +} FlutterSemanticsCustomAction; + +typedef void (*FlutterUpdateSemanticsNodeCallback)( + const FlutterSemanticsNode* /* semantics node */, + void* /* user data */); + +typedef void (*FlutterUpdateSemanticsCustomActionCallback)( + const FlutterSemanticsCustomAction* /* semantics custom action */, + void* /* user data */); + typedef struct { // The size of this struct. Must be sizeof(FlutterProjectArgs). size_t struct_size; @@ -268,6 +494,17 @@ typedef struct { // The callback invoked by the engine in root isolate scope. Called // immediately after the root isolate has been created and marked runnable. VoidCallback root_isolate_create_callback; + // The callback invoked by the engine in order to give the embedder the + // chance to respond to semantics node updates from the Dart application. The + // callback will be invoked on the thread on which the |FlutterEngineRun| + // call is made. + FlutterUpdateSemanticsNodeCallback update_semantics_node_callback; + // The callback invoked by the engine in order to give the embedder the + // chance to respond to updates to semantics custom actions from the Dart + // application. The callback will be invoked on the thread on which the + // |FlutterEngineRun| call is made. + FlutterUpdateSemanticsCustomActionCallback + update_semantics_custom_action_callback; } FlutterProjectArgs; FLUTTER_EXPORT @@ -331,6 +568,30 @@ FlutterEngineResult FlutterEngineMarkExternalTextureFrameAvailable( FlutterEngine engine, int64_t texture_identifier); +// Enable or disable accessibility semantics. +// +// When enabled, changes to the semantic contents of the window are sent via +// the |FlutterUpdateSemanticsNodeCallback| registered to +// |update_semantics_node_callback| in |FlutterProjectArgs|; +FLUTTER_EXPORT +FlutterEngineResult FlutterEngineUpdateSemanticsEnabled(FlutterEngine engine, + bool enabled); + +// Sets additional accessibility features. +FLUTTER_EXPORT +FlutterEngineResult FlutterEngineUpdateAccessibilityFeatures( + FlutterEngine engine, + FlutterAccessibilityFeature features); + +// Dispatch a semantics action to the specified semantics node. +FLUTTER_EXPORT +FlutterEngineResult FlutterEngineDispatchSemanticsAction( + FlutterEngine engine, + uint64_t id, + FlutterSemanticsAction action, + const uint8_t* data, + size_t data_length); + #if defined(__cplusplus) } // extern "C" #endif diff --git a/shell/platform/embedder/embedder_engine.cc b/shell/platform/embedder/embedder_engine.cc index 168de1b83..0f2127587 100644 --- a/shell/platform/embedder/embedder_engine.cc +++ b/shell/platform/embedder/embedder_engine.cc @@ -146,4 +146,49 @@ bool EmbedderEngine::MarkTextureFrameAvailable(int64_t texture) { return true; } +bool EmbedderEngine::SetSemanticsEnabled(bool enabled) { + if (!IsValid()) { + return false; + } + shell_->GetTaskRunners().GetUITaskRunner()->PostTask( + [engine = shell_->GetEngine(), enabled] { + if (engine) { + engine->SetSemanticsEnabled(enabled); + } + }); + return true; +} + +bool EmbedderEngine::SetAccessibilityFeatures(int32_t flags) { + if (!IsValid()) { + return false; + } + shell_->GetTaskRunners().GetUITaskRunner()->PostTask( + [engine = shell_->GetEngine(), flags] { + if (engine) { + engine->SetAccessibilityFeatures(flags); + } + }); + return true; +} + +bool EmbedderEngine::DispatchSemanticsAction(int id, + blink::SemanticsAction action, + std::vector args) { + if (!IsValid()) { + return false; + } + shell_->GetTaskRunners().GetUITaskRunner()->PostTask( + fml::MakeCopyable([engine = shell_->GetEngine(), // engine + id, // id + action, // action + args = std::move(args) // args + ]() mutable { + if (engine) { + engine->DispatchSemanticsAction(id, action, std::move(args)); + } + })); + return true; +} + } // namespace shell diff --git a/shell/platform/embedder/embedder_engine.h b/shell/platform/embedder/embedder_engine.h index 204776554..f5c439901 100644 --- a/shell/platform/embedder/embedder_engine.h +++ b/shell/platform/embedder/embedder_engine.h @@ -11,6 +11,7 @@ #include "flutter/shell/common/shell.h" #include "flutter/shell/common/thread_host.h" #include "flutter/shell/platform/embedder/embedder.h" +#include "flutter/shell/platform/embedder/embedder_engine.h" #include "flutter/shell/platform/embedder/embedder_external_texture_gl.h" namespace shell { @@ -50,6 +51,14 @@ class EmbedderEngine { bool MarkTextureFrameAvailable(int64_t texture); + bool SetSemanticsEnabled(bool enabled); + + bool SetAccessibilityFeatures(int32_t flags); + + bool DispatchSemanticsAction(int id, + blink::SemanticsAction action, + std::vector args); + private: const ThreadHost thread_host_; std::unique_ptr shell_; diff --git a/shell/platform/embedder/fixtures/a11y_main.dart b/shell/platform/embedder/fixtures/a11y_main.dart new file mode 100644 index 000000000..a0315f56a --- /dev/null +++ b/shell/platform/embedder/fixtures/a11y_main.dart @@ -0,0 +1,112 @@ +import 'dart:async'; +import 'dart:typed_data'; +import 'dart:ui'; + +Float64List kIdentityTransform = () { + final Float64List values = Float64List(16); + values[0] = 1.0; + values[5] = 1.0; + values[10] = 1.0; + values[15] = 1.0; + return values; +}(); + +void signalNativeTest() native 'SignalNativeTest'; +void notifySemanticsEnabled(bool enabled) native 'NotifyTestData1'; +void notifyAccessibilityFeatures(bool reduceMotion) native 'NotifyTestData1'; +void notifySemanticsAction(int nodeId, int action, List data) native 'NotifyTestData3'; + +/// Returns a future that completes when `window.onSemanticsEnabledChanged` +/// fires. +Future get semanticsChanged { + final Completer semanticsChanged = Completer(); + window.onSemanticsEnabledChanged = semanticsChanged.complete; + return semanticsChanged.future; +} + +/// Returns a future that completes when `window.onAccessibilityFeaturesChanged` +/// fires. +Future get accessibilityFeaturesChanged { + final Completer featuresChanged = Completer(); + window.onAccessibilityFeaturesChanged = featuresChanged.complete; + return featuresChanged.future; +} + +class SemanticsActionData { + const SemanticsActionData(this.id, this.action, this.args); + final int id; + final SemanticsAction action; + final ByteData args; +} + +Future get semanticsAction { + final Completer actionReceived = Completer(); + window.onSemanticsAction = (int id, SemanticsAction action, ByteData args) { + actionReceived.complete(SemanticsActionData(id, action, args)); + }; + return actionReceived.future; +} + +main() async { + // Return initial state (semantics disabled). + notifySemanticsEnabled(window.semanticsEnabled); + + // Await semantics enabled from embedder. + await semanticsChanged; + notifySemanticsEnabled(window.semanticsEnabled); + + // Return initial state of accessibility features. + notifyAccessibilityFeatures(window.accessibilityFeatures.reduceMotion); + + // Await accessibility features changed from embedder. + await accessibilityFeaturesChanged; + notifyAccessibilityFeatures(window.accessibilityFeatures.reduceMotion); + + // Fire semantics update. + final SemanticsUpdateBuilder builder = SemanticsUpdateBuilder() + ..updateNode( + id: 42, + label: 'A: root', + rect: Rect.fromLTRB(0.0, 0.0, 10.0, 10.0), + transform: kIdentityTransform, + childrenInTraversalOrder: Int32List.fromList([84, 96]), + childrenInHitTestOrder: Int32List.fromList([96, 84]), + ) + ..updateNode( + id: 84, + label: 'B: leaf', + rect: Rect.fromLTRB(40.0, 40.0, 80.0, 80.0), + transform: kIdentityTransform, + ) + ..updateNode( + id: 96, + label: 'C: branch', + rect: Rect.fromLTRB(40.0, 40.0, 80.0, 80.0), + transform: kIdentityTransform, + childrenInTraversalOrder: Int32List.fromList([128]), + childrenInHitTestOrder: Int32List.fromList([128]), + ) + ..updateNode( + id: 128, + label: 'D: leaf', + rect: Rect.fromLTRB(40.0, 40.0, 80.0, 80.0), + transform: kIdentityTransform, + additionalActions: Int32List.fromList([21]), + ) + ..updateCustomAction( + id: 21, + label: 'Archive', + hint: 'archive message', + ); + window.updateSemantics(builder.build()); + signalNativeTest(); + + // Await semantics action from embedder. + final SemanticsActionData data = await semanticsAction; + final List actionArgs = [data.args.getInt8(0), data.args.getInt8(1)]; + notifySemanticsAction(data.id, data.action.index, actionArgs); + + // Await semantics disabled from embedder. + await semanticsChanged; + notifySemanticsEnabled(window.semanticsEnabled); +} diff --git a/shell/platform/embedder/platform_view_embedder.cc b/shell/platform/embedder/platform_view_embedder.cc index 5a5551b75..736a54532 100644 --- a/shell/platform/embedder/platform_view_embedder.cc +++ b/shell/platform/embedder/platform_view_embedder.cc @@ -30,6 +30,19 @@ PlatformViewEmbedder::PlatformViewEmbedder( PlatformViewEmbedder::~PlatformViewEmbedder() = default; +void PlatformViewEmbedder::UpdateSemantics( + blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) { + if (platform_dispatch_table_.update_semantics_nodes_callback != nullptr) { + platform_dispatch_table_.update_semantics_nodes_callback(std::move(update)); + } + if (platform_dispatch_table_.update_semantics_custom_actions_callback != + nullptr) { + platform_dispatch_table_.update_semantics_custom_actions_callback( + std::move(actions)); + } +} + void PlatformViewEmbedder::HandlePlatformMessage( fml::RefPtr message) { if (!message) { diff --git a/shell/platform/embedder/platform_view_embedder.h b/shell/platform/embedder/platform_view_embedder.h index bea840387..b57158d95 100644 --- a/shell/platform/embedder/platform_view_embedder.h +++ b/shell/platform/embedder/platform_view_embedder.h @@ -18,10 +18,17 @@ namespace shell { class PlatformViewEmbedder final : public PlatformView { public: + using UpdateSemanticsNodesCallback = + std::function; + using UpdateSemanticsCustomActionsCallback = + std::function; using PlatformMessageResponseCallback = std::function)>; struct PlatformDispatchTable { + UpdateSemanticsNodesCallback update_semantics_nodes_callback; // optional + UpdateSemanticsCustomActionsCallback + update_semantics_custom_actions_callback; // optional PlatformMessageResponseCallback platform_message_response_callback; // optional }; @@ -42,6 +49,11 @@ class PlatformViewEmbedder final : public PlatformView { ~PlatformViewEmbedder() override; + // |shell::PlatformView| + void UpdateSemantics( + blink::SemanticsNodeUpdates update, + blink::CustomAccessibilityActionUpdates actions) override; + // |shell::PlatformView| void HandlePlatformMessage( fml::RefPtr message) override; diff --git a/shell/platform/embedder/tests/embedder_a11y_unittests.cc b/shell/platform/embedder/tests/embedder_a11y_unittests.cc new file mode 100644 index 000000000..511a1dcf2 --- /dev/null +++ b/shell/platform/embedder/tests/embedder_a11y_unittests.cc @@ -0,0 +1,205 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Allow access to fml::MessageLoop::GetCurrent() in order to flush platform +// thread tasks. +#define FML_USED_ON_EMBEDDER + +#include +#include "embedder.h" +#include "flutter/fml/macros.h" +#include "flutter/fml/message_loop.h" +#include "flutter/fml/synchronization/waitable_event.h" +#include "flutter/lib/ui/semantics/semantics_node.h" +#include "flutter/shell/platform/embedder/embedder_engine.h" +#include "flutter/testing/testing.h" +#include "third_party/dart/runtime/include/dart_api.h" +#include "third_party/tonic/converter/dart_converter.h" +#include "third_party/tonic/dart_library_natives.h" + +#define REGISTER_FUNCTION(name, count) {"" #name, name, count, true}, +#define DECLARE_FUNCTION(name, count) \ + extern void name(Dart_NativeArguments args); +#define BUILTIN_NATIVE_LIST(V) \ + V(SignalNativeTest, 0) \ + V(NotifyTestData1, 1) \ + V(NotifyTestData3, 3) + +BUILTIN_NATIVE_LIST(DECLARE_FUNCTION); + +static tonic::DartLibraryNatives* g_natives; + +Dart_NativeFunction GetNativeFunction(Dart_Handle name, + int argument_count, + bool* auto_setup_scope) { + return g_natives->GetNativeFunction(name, argument_count, auto_setup_scope); +} + +const uint8_t* GetSymbol(Dart_NativeFunction native_function) { + return g_natives->GetSymbol(native_function); +} + +using OnTestDataCallback = std::function; + +fml::AutoResetWaitableEvent g_latch; +OnTestDataCallback g_test_data_callback = [](Dart_NativeArguments) {}; + +// Called by the Dart text fixture on the UI thread to signal that the C++ +// unittest should resume. +void SignalNativeTest(Dart_NativeArguments args) { + g_latch.Signal(); +} + +// Called by test fixture on UI thread to pass data back to this test. +// 1 parameter version. +void NotifyTestData1(Dart_NativeArguments args) { + g_test_data_callback(args); +} + +// Called by test fixture on UI thread to pass data back to this test. +// 3 parameter version. +void NotifyTestData3(Dart_NativeArguments args) { + g_test_data_callback(args); +} + +TEST(EmbedderTest, CanLaunchAndShutdownWithValidProjectArgs) { + FlutterSoftwareRendererConfig renderer; + renderer.struct_size = sizeof(FlutterSoftwareRendererConfig); + renderer.surface_present_callback = [](void*, const void*, size_t, size_t) { + return false; + }; + + FlutterRendererConfig config = {}; + config.type = FlutterRendererType::kSoftware; + config.software = renderer; + + FlutterProjectArgs args = {}; + args.struct_size = sizeof(FlutterProjectArgs); + args.assets_path = testing::GetFixturesPath(); + + // Register native functions to be called from test fixture. + g_natives = new tonic::DartLibraryNatives(); + g_natives->Register({BUILTIN_NATIVE_LIST(REGISTER_FUNCTION)}); + args.root_isolate_create_callback = [](void*) { + Dart_SetNativeResolver(Dart_RootLibrary(), GetNativeFunction, GetSymbol); + }; + + typedef struct { + std::function on_semantics_update; + std::function + on_custom_action_update; + } TestData; + auto test_data = TestData{}; + args.update_semantics_node_callback = [](const FlutterSemanticsNode* node, + void* data) { + auto test_data = reinterpret_cast(data); + test_data->on_semantics_update(node); + }; + args.update_semantics_custom_action_callback = + [](const FlutterSemanticsCustomAction* action, void* data) { + auto test_data = reinterpret_cast(data); + test_data->on_custom_action_update(action); + }; + + // Start the engine, run text fixture. + FlutterEngine engine = nullptr; + FlutterEngineResult result = FlutterEngineRun(FLUTTER_ENGINE_VERSION, &config, + &args, &test_data, &engine); + ASSERT_EQ(result, FlutterEngineResult::kSuccess); + + // Wait for initial NotifySemanticsEnabled(false). + g_test_data_callback = [](Dart_NativeArguments args) { + bool enabled; + Dart_GetNativeBooleanArgument(args, 0, &enabled); + ASSERT_FALSE(enabled); + g_latch.Signal(); + }; + g_latch.Wait(); + + // Enable semantics. Wait for NotifySemanticsEnabled(true). + g_test_data_callback = [](Dart_NativeArguments args) { + bool enabled; + Dart_GetNativeBooleanArgument(args, 0, &enabled); + ASSERT_TRUE(enabled); + g_latch.Signal(); + }; + result = FlutterEngineUpdateSemanticsEnabled(engine, true); + ASSERT_EQ(result, FlutterEngineResult::kSuccess); + g_latch.Wait(); + + // Wait for initial accessibility features (reduce_motion == false) + g_test_data_callback = [](Dart_NativeArguments args) { + bool enabled; + Dart_GetNativeBooleanArgument(args, 0, &enabled); + ASSERT_FALSE(enabled); + g_latch.Signal(); + }; + g_latch.Wait(); + + // Set accessibility features: (reduce_motion == true) + g_test_data_callback = [](Dart_NativeArguments args) { + bool enabled; + Dart_GetNativeBooleanArgument(args, 0, &enabled); + ASSERT_TRUE(enabled); + g_latch.Signal(); + }; + result = FlutterEngineUpdateAccessibilityFeatures( + engine, kFlutterAccessibilityFeatureReduceMotion); + ASSERT_EQ(result, FlutterEngineResult::kSuccess); + g_latch.Wait(); + + // Wait for UpdateSemantics callback on platform (current) thread. + int node_count = 0; + test_data.on_semantics_update = + [&node_count](const FlutterSemanticsNode* node) { ++node_count; }; + int action_count = 0; + test_data.on_custom_action_update = + [&action_count](const FlutterSemanticsCustomAction* action) { + ++action_count; + }; + g_latch.Wait(); + fml::MessageLoop::GetCurrent().RunExpiredTasksNow(); + ASSERT_EQ(4, node_count); + ASSERT_EQ(1, action_count); + + // Dispatch a tap to semantics node 42. Wait for NotifySemanticsAction. + g_test_data_callback = [](Dart_NativeArguments args) { + int64_t node_id; + Dart_GetNativeIntegerArgument(args, 0, &node_id); + ASSERT_EQ(42, node_id); + + int64_t action_id; + Dart_GetNativeIntegerArgument(args, 1, &action_id); + ASSERT_EQ(static_cast(blink::SemanticsAction::kTap), action_id); + + Dart_Handle semantic_args = Dart_GetNativeArgument(args, 2); + int64_t data; + Dart_Handle dart_int = Dart_ListGetAt(semantic_args, 0); + Dart_IntegerToInt64(dart_int, &data); + ASSERT_EQ(2, data); + + dart_int = Dart_ListGetAt(semantic_args, 1); + Dart_IntegerToInt64(dart_int, &data); + ASSERT_EQ(1, data); + g_latch.Signal(); + }; + std::vector bytes({2, 1}); + result = FlutterEngineDispatchSemanticsAction( + engine, 42, kFlutterSemanticsActionTap, &bytes[0], bytes.size()); + g_latch.Wait(); + + // Disable semantics. Wait for NotifySemanticsEnabled(false). + g_test_data_callback = [](Dart_NativeArguments args) { + bool enabled; + Dart_GetNativeBooleanArgument(args, 0, &enabled); + ASSERT_FALSE(enabled); + g_latch.Signal(); + }; + result = FlutterEngineUpdateSemanticsEnabled(engine, false); + ASSERT_EQ(result, FlutterEngineResult::kSuccess); + g_latch.Wait(); + + result = FlutterEngineShutdown(engine); + ASSERT_EQ(result, FlutterEngineResult::kSuccess); +} -- GitLab