Notify Win32FlutterWindow of cursor updates (#23795)

During multi-step text input composing, such as with Chinese, Japanese,
and Korean text input, the framework sends embedders cursor rect updates
in the form of two messages:

* TextInput.setMarkedTextRect: notifies the embedder the size and
  position of the composing text rect (or cursor when not composing) in
  local coordinates.
* TextInput.setEditableSizeAndTransform: notifies the embedder of the
  size of the EditableText and 4x4 transform matrix from local to
  PipelineOwner.rootNode coordinates.

On receipt of either message, we cache a local copy on the
TextInputPlugin and notify the Win32FlutterWindow of the updated cursor
rect. In a followup patch, we update Win32FlutterWindow to implement the
Win32 input manager (IMM) calls required to position the IME candidates
window while editing.

ci/licenses_golden/licenses_flutter

@@ -917,6 +917,8 @@ FILE: ../../../flutter/shell/platform/common/cpp/engine_switches_unittests.cc
 FILE: ../../../flutter/shell/platform/common/cpp/flutter_platform_node_delegate.cc
 FILE: ../../../flutter/shell/platform/common/cpp/flutter_platform_node_delegate.h
 FILE: ../../../flutter/shell/platform/common/cpp/flutter_platform_node_delegate_unittests.cc
+FILE: ../../../flutter/shell/platform/common/cpp/geometry.h
+FILE: ../../../flutter/shell/platform/common/cpp/geometry_unittests.cc
 FILE: ../../../flutter/shell/platform/common/cpp/incoming_message_dispatcher.cc
 FILE: ../../../flutter/shell/platform/common/cpp/incoming_message_dispatcher.h
 FILE: ../../../flutter/shell/platform/common/cpp/json_message_codec.cc
@@ -1496,6 +1498,7 @@ FILE: ../../../flutter/shell/platform/windows/task_runner_winuwp.cc
 FILE: ../../../flutter/shell/platform/windows/task_runner_winuwp.h
 FILE: ../../../flutter/shell/platform/windows/text_input_plugin.cc
 FILE: ../../../flutter/shell/platform/windows/text_input_plugin.h
+FILE: ../../../flutter/shell/platform/windows/text_input_plugin_delegate.h
 FILE: ../../../flutter/shell/platform/windows/win32_dpi_utils.cc
 FILE: ../../../flutter/shell/platform/windows/win32_dpi_utils.h
 FILE: ../../../flutter/shell/platform/windows/win32_dpi_utils_unittests.cc

shell/platform/common/cpp/BUILD.gn

@@ -124,7 +124,10 @@ source_set("common_cpp") {
 # embedding is futher along and it's clearer how much, if any, shared
 # API surface there will be.
 source_set("common_cpp_core") {
-  public = [ "path_utils.h" ]
+  public = [
+    "geometry.h",
+    "path_utils.h",
+  ]
 
   sources = [ "path_utils.cc" ]
 
@@ -160,6 +163,7 @@ if (enable_unittests) {
 
     sources = [
       "engine_switches_unittests.cc",
+      "geometry_unittests.cc",
       "json_message_codec_unittests.cc",
       "json_method_codec_unittests.cc",
       "text_input_model_unittests.cc",

shell/platform/common/cpp/geometry.h

+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef FLUTTER_SHELL_PLATFORM_COMMON_CPP_GEOMETRY_H_
+#define FLUTTER_SHELL_PLATFORM_COMMON_CPP_GEOMETRY_H_
+
+#include <cmath>
+
+namespace flutter {
+
+// A point in Cartesian space relative to a separately-maintained origin.
+class Point {
+ public:
+  Point() = default;
+  Point(double x, double y) : x_(x), y_(y) {}
+  Point(const Point& point) = default;
+  Point& operator=(const Point& other) = default;
+
+  double x() const { return x_; }
+  double y() const { return y_; }
+
+  bool operator==(const Point& other) const {
+    return x_ == other.x_ && y_ == other.y_;
+  }
+
+ private:
+  double x_ = 0.0;
+  double y_ = 0.0;
+};
+
+// A 2D floating-point size with non-negative dimensions.
+class Size {
+ public:
+  Size() = default;
+  Size(double width, double height)
+      : width_(std::fmax(0.0, width)), height_(std::fmax(0.0, height)) {}
+
+  Size(const Size& size) = default;
+  Size& operator=(const Size& other) = default;
+
+  double width() const { return width_; }
+  double height() const { return height_; }
+
+  bool operator==(const Size& other) const {
+    return width_ == other.width_ && height_ == other.height_;
+  }
+
+ private:
+  double width_ = 0.0;
+  double height_ = 0.0;
+};
+
+// A rectangle with position in Cartesian space specified relative to a
+// separately-maintained origin.
+class Rect {
+ public:
+  Rect() = default;
+  Rect(const Point& origin, const Size& size) : origin_(origin), size_(size) {}
+  Rect(const Rect& rect) = default;
+  Rect& operator=(const Rect& other) = default;
+
+  double left() const { return origin_.x(); }
+  double top() const { return origin_.y(); }
+  double right() const { return origin_.x() + size_.width(); }
+  double bottom() const { return origin_.y() + size_.height(); }
+  double width() const { return size_.width(); }
+  double height() const { return size_.height(); }
+  Point origin() const { return origin_; }
+  Size size() const { return size_; }
+
+  bool operator==(const Rect& other) const {
+    return origin_ == other.origin_ && size_ == other.size_;
+  }
+
+ private:
+  Point origin_;
+  Size size_;
+};
+
+}  // namespace flutter
+
+#endif  // FLUTTER_SHELL_PLATFORM_COMMON_CPP_GEOMETRY_H_

shell/platform/common/cpp/geometry_unittests.cc

+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "flutter/shell/platform/common/cpp/geometry.h"
+
+#include "gtest/gtest.h"
+
+namespace flutter {
+
+TEST(Point, SetsCoordinates) {
+  Point point(-30.0, 42.0);
+  EXPECT_DOUBLE_EQ(-30.0, point.x());
+  EXPECT_DOUBLE_EQ(42.0, point.y());
+}
+
+TEST(Size, SetsDimensions) {
+  Size size(20.0, 42.0);
+  EXPECT_DOUBLE_EQ(20.0, size.width());
+  EXPECT_DOUBLE_EQ(42.0, size.height());
+}
+
+TEST(Size, ClampsDimensionsPositive) {
+  Size size(-20.0, -42.0);
+  EXPECT_DOUBLE_EQ(0.0, size.width());
+  EXPECT_DOUBLE_EQ(0.0, size.height());
+}
+
+TEST(Rect, SetsOriginAndSize) {
+  Point origin(-30.0, 42.0);
+  Size size(20.0, 22.0);
+  Rect rect(origin, size);
+  EXPECT_EQ(origin, rect.origin());
+  EXPECT_EQ(size, rect.size());
+}
+
+TEST(Rect, ReturnsLTRB) {
+  Point origin(-30.0, 42.0);
+  Size size(20.0, 22.0);
+  Rect rect(origin, size);
+  EXPECT_DOUBLE_EQ(-30.0, rect.left());
+  EXPECT_DOUBLE_EQ(42.0, rect.top());
+  EXPECT_DOUBLE_EQ(-10.0, rect.right());
+  EXPECT_DOUBLE_EQ(64.0, rect.bottom());
+}
+
+TEST(Rect, ReturnsWidthHeight) {
+  Point origin(-30.0, 42.0);
+  Size size(20.0, 22.0);
+  Rect rect(origin, size);
+  EXPECT_DOUBLE_EQ(20.0, rect.width());
+  EXPECT_DOUBLE_EQ(22.0, rect.height());
+}
+
+}  // namespace flutter

shell/platform/windows/flutter_windows_view.cc

@@ -37,8 +37,8 @@ void FlutterWindowsView::SetEngine(
   auto internal_plugin_messenger = internal_plugin_registrar_->messenger();
   keyboard_hook_handlers_.push_back(
       std::make_unique<flutter::KeyEventHandler>(internal_plugin_messenger));
-  keyboard_hook_handlers_.push_back(
-      std::make_unique<flutter::TextInputPlugin>(internal_plugin_messenger));
+  keyboard_hook_handlers_.push_back(std::make_unique<flutter::TextInputPlugin>(
+      internal_plugin_messenger, this));
   platform_handler_ = PlatformHandler::Create(internal_plugin_messenger, this);
   cursor_handler_ = std::make_unique<flutter::CursorHandler>(
       internal_plugin_messenger, binding_handler_.get());
@@ -135,6 +135,10 @@ void FlutterWindowsView::OnScroll(double x,
   SendScroll(x, y, delta_x, delta_y, scroll_offset_multiplier);
 }
 
+void FlutterWindowsView::OnCursorRectUpdated(const Rect& rect) {
+  binding_handler_->UpdateCursorRect(rect);
+}
+
 // Sends new size  information to FlutterEngine.
 void FlutterWindowsView::SendWindowMetrics(size_t width,
                                            size_t height,
@@ -160,12 +164,15 @@ void FlutterWindowsView::SetEventPhaseFromCursorButtonState(
     FlutterPointerEvent* event_data) const {
   // For details about this logic, see FlutterPointerPhase in the embedder.h
   // file.
-  event_data->phase =
-      mouse_state_.buttons == 0
-          ? mouse_state_.flutter_state_is_down ? FlutterPointerPhase::kUp
-                                               : FlutterPointerPhase::kHover
-          : mouse_state_.flutter_state_is_down ? FlutterPointerPhase::kMove
-                                               : FlutterPointerPhase::kDown;
+  if (mouse_state_.buttons == 0) {
+    event_data->phase = mouse_state_.flutter_state_is_down
+                            ? FlutterPointerPhase::kUp
+                            : FlutterPointerPhase::kHover;
+  } else {
+    event_data->phase = mouse_state_.flutter_state_is_down
+                            ? FlutterPointerPhase::kMove
+                            : FlutterPointerPhase::kDown;
+  }
 }
 
 void FlutterWindowsView::SendPointerMove(double x, double y) {

shell/platform/windows/flutter_windows_view.h

@@ -13,6 +13,7 @@
 #include <vector>
 
 #include "flutter/shell/platform/common/cpp/client_wrapper/include/flutter/plugin_registrar.h"
+#include "flutter/shell/platform/common/cpp/geometry.h"
 #include "flutter/shell/platform/embedder/embedder.h"
 #include "flutter/shell/platform/windows/angle_surface_manager.h"
 #include "flutter/shell/platform/windows/cursor_handler.h"
@@ -22,6 +23,7 @@
 #include "flutter/shell/platform/windows/platform_handler.h"
 #include "flutter/shell/platform/windows/public/flutter_windows.h"
 #include "flutter/shell/platform/windows/text_input_plugin.h"
+#include "flutter/shell/platform/windows/text_input_plugin_delegate.h"
 #include "flutter/shell/platform/windows/window_binding_handler.h"
 #include "flutter/shell/platform/windows/window_binding_handler_delegate.h"
 #include "flutter/shell/platform/windows/window_state.h"
@@ -33,7 +35,8 @@ inline constexpr uint32_t kWindowFrameBufferID = 0;
 
 // An OS-windowing neutral abstration for flutter
 // view that works with win32 hwnds and Windows::UI::Composition visuals.
-class FlutterWindowsView : public WindowBindingHandlerDelegate {
+class FlutterWindowsView : public WindowBindingHandlerDelegate,
+                           public TextInputPluginDelegate {
  public:
   // Creates a FlutterWindowsView with the given implementator of
   // WindowBindingHandler.
@@ -106,6 +109,9 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate {
                 double delta_y,
                 int scroll_offset_multiplier) override;
 
+  // |TextInputPluginDelegate|
+  void OnCursorRectUpdated(const Rect& rect) override;
+
  private:
   // Struct holding the mouse state. The engine doesn't keep track of which
   // mouse buttons have been pressed, so it's the embedding's responsibility.

shell/platform/windows/text_input_plugin.cc

@@ -10,12 +10,16 @@
 #include <iostream>
 
 #include "flutter/shell/platform/common/cpp/json_method_codec.h"
+#include "flutter/shell/platform/windows/flutter_windows_view.h"
 
 static constexpr char kSetEditingStateMethod[] = "TextInput.setEditingState";
 static constexpr char kClearClientMethod[] = "TextInput.clearClient";
 static constexpr char kSetClientMethod[] = "TextInput.setClient";
 static constexpr char kShowMethod[] = "TextInput.show";
 static constexpr char kHideMethod[] = "TextInput.hide";
+static constexpr char kSetMarkedTextRect[] = "TextInput.setMarkedTextRect";
+static constexpr char kSetEditableSizeAndTransform[] =
+    "TextInput.setEditableSizeAndTransform";
 
 static constexpr char kMultilineInputType[] = "TextInputType.multiline";
 
@@ -34,6 +38,11 @@ static constexpr char kSelectionBaseKey[] = "selectionBase";
 static constexpr char kSelectionExtentKey[] = "selectionExtent";
 static constexpr char kSelectionIsDirectionalKey[] = "selectionIsDirectional";
 static constexpr char kTextKey[] = "text";
+static constexpr char kXKey[] = "x";
+static constexpr char kYKey[] = "y";
+static constexpr char kWidthKey[] = "width";
+static constexpr char kHeightKey[] = "height";
+static constexpr char kTransformKey[] = "transform";
 
 static constexpr char kChannelName[] = "flutter/textinput";
 
@@ -73,11 +82,13 @@ void TextInputPlugin::KeyboardHook(FlutterWindowsView* view,
   }
 }
 
-TextInputPlugin::TextInputPlugin(flutter::BinaryMessenger* messenger)
+TextInputPlugin::TextInputPlugin(flutter::BinaryMessenger* messenger,
+                                 TextInputPluginDelegate* delegate)
     : channel_(std::make_unique<flutter::MethodChannel<rapidjson::Document>>(
           messenger,
           kChannelName,
           &flutter::JsonMethodCodec::GetInstance())),
+      delegate_(delegate),
       active_model_(nullptr) {
   channel_->SetMethodCallHandler(
       [this](
@@ -171,6 +182,54 @@ void TextInputPlugin::HandleMethodCall(
     }
     active_model_->SetText(text->value.GetString());
     active_model_->SetSelection(TextRange(base, extent));
+  } else if (method.compare(kSetMarkedTextRect) == 0) {
+    if (!method_call.arguments() || method_call.arguments()->IsNull()) {
+      result->Error(kBadArgumentError, "Method invoked without args");
+      return;
+    }
+    const rapidjson::Document& args = *method_call.arguments();
+    auto x = args.FindMember(kXKey);
+    auto y = args.FindMember(kYKey);
+    auto width = args.FindMember(kWidthKey);
+    auto height = args.FindMember(kHeightKey);
+    if (x == args.MemberEnd() || x->value.IsNull() ||          //
+        y == args.MemberEnd() || y->value.IsNull() ||          //
+        width == args.MemberEnd() || width->value.IsNull() ||  //
+        height == args.MemberEnd() || height->value.IsNull()) {
+      result->Error(kInternalConsistencyError,
+                    "Composing rect values invalid.");
+      return;
+    }
+    composing_rect_ = {{x->value.GetDouble(), y->value.GetDouble()},
+                       {width->value.GetDouble(), height->value.GetDouble()}};
+
+    Rect transformed_rect = GetCursorRect();
+    delegate_->OnCursorRectUpdated(transformed_rect);
+  } else if (method.compare(kSetEditableSizeAndTransform) == 0) {
+    if (!method_call.arguments() || method_call.arguments()->IsNull()) {
+      result->Error(kBadArgumentError, "Method invoked without args");
+      return;
+    }
+    const rapidjson::Document& args = *method_call.arguments();
+    auto transform = args.FindMember(kTransformKey);
+    if (transform == args.MemberEnd() || transform->value.IsNull() ||
+        !transform->value.IsArray() || transform->value.Size() != 16) {
+      result->Error(kInternalConsistencyError,
+                    "EditableText transform invalid.");
+      return;
+    }
+    size_t i = 0;
+    for (auto& entry : transform->value.GetArray()) {
+      if (entry.IsNull()) {
+        result->Error(kInternalConsistencyError,
+                      "EditableText transform contains null value.");
+        return;
+      }
+      editabletext_transform_[i / 4][i % 4] = entry.GetDouble();
+      ++i;
+    }
+    Rect transformed_rect = GetCursorRect();
+    delegate_->OnCursorRectUpdated(transformed_rect);
   } else {
     result->NotImplemented();
     return;
@@ -180,6 +239,17 @@ void TextInputPlugin::HandleMethodCall(
   result->Success();
 }
 
+Rect TextInputPlugin::GetCursorRect() const {
+  Point transformed_point = {
+      composing_rect_.left() * editabletext_transform_[0][0] +
+          composing_rect_.top() * editabletext_transform_[1][0] +
+          editabletext_transform_[3][0] + composing_rect_.width(),
+      composing_rect_.left() * editabletext_transform_[0][1] +
+          composing_rect_.top() * editabletext_transform_[1][1] +
+          editabletext_transform_[3][1] + composing_rect_.height()};
+  return {transformed_point, composing_rect_.size()};
+}
+
 void TextInputPlugin::SendStateUpdate(const TextInputModel& model) {
   auto args = std::make_unique<rapidjson::Document>(rapidjson::kArrayType);
   auto& allocator = args->GetAllocator();

shell/platform/windows/text_input_plugin.h

@@ -5,15 +5,18 @@
 #ifndef FLUTTER_SHELL_PLATFORM_WINDOWS_TEXT_INPUT_PLUGIN_H_
 #define FLUTTER_SHELL_PLATFORM_WINDOWS_TEXT_INPUT_PLUGIN_H_
 
+#include <array>
 #include <map>
 #include <memory>
 
 #include "flutter/shell/platform/common/cpp/client_wrapper/include/flutter/binary_messenger.h"
 #include "flutter/shell/platform/common/cpp/client_wrapper/include/flutter/method_channel.h"
+#include "flutter/shell/platform/common/cpp/geometry.h"
 #include "flutter/shell/platform/common/cpp/json_method_codec.h"
 #include "flutter/shell/platform/common/cpp/text_input_model.h"
 #include "flutter/shell/platform/windows/keyboard_hook_handler.h"
 #include "flutter/shell/platform/windows/public/flutter_windows.h"
+#include "flutter/shell/platform/windows/text_input_plugin_delegate.h"
 
 namespace flutter {
 
@@ -24,7 +27,8 @@ class FlutterWindowsView;
 // Specifically handles window events within windows.
 class TextInputPlugin : public KeyboardHookHandler {
  public:
-  explicit TextInputPlugin(flutter::BinaryMessenger* messenger);
+  explicit TextInputPlugin(flutter::BinaryMessenger* messenger,
+                           TextInputPluginDelegate* delegate);
 
   virtual ~TextInputPlugin();
 
@@ -50,9 +54,16 @@ class TextInputPlugin : public KeyboardHookHandler {
       const flutter::MethodCall<rapidjson::Document>& method_call,
       std::unique_ptr<flutter::MethodResult<rapidjson::Document>> result);
 
+  // Returns the composing rect, or if IME composing mode is not active, the
+  // cursor rect in the PipelineOwner root coordinate system.
+  Rect GetCursorRect() const;
+
   // The MethodChannel used for communication with the Flutter engine.
   std::unique_ptr<flutter::MethodChannel<rapidjson::Document>> channel_;
 
+  // The associated |TextInputPluginDelegate|.
+  TextInputPluginDelegate* delegate_;
+
   // The active client id.
   int client_id_;
 
@@ -66,6 +77,20 @@ class TextInputPlugin : public KeyboardHookHandler {
   // An action requested by the user on the input client. See available options:
   // https://api.flutter.dev/flutter/services/TextInputAction-class.html
   std::string input_action_;
+
+  // The smallest rect, in local coordinates, of the text in the composing
+  // range, or of the caret in the case where there is no current composing
+  // range. This value is updated via `TextInput.setMarkedTextRect` messages
+  // over the text input channel.
+  Rect composing_rect_;
+
+  // A 4x4 matrix that maps from `EditableText` local coordinates to the
+  // coordinate system of `PipelineOwner.rootNode`.
+  std::array<std::array<double, 4>, 4> editabletext_transform_ = {
+      0.0, 0.0, 0.0, 0.0,  //
+      0.0, 0.0, 0.0, 0.0,  //
+      0.0, 0.0, 0.0, 0.0,  //
+      0.0, 0.0, 0.0, 0.0};
 };
 
 }  // namespace flutter

shell/platform/windows/text_input_plugin_delegate.h

+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef FLUTTER_SHELL_PLATFORM_WINDOWS_TEXT_INPUT_PLUGIN_DELEGATE_H_
+#define FLUTTER_SHELL_PLATFORM_WINDOWS_TEXT_INPUT_PLUGIN_DELEGATE_H_
+
+#include "flutter/shell/platform/common/cpp/geometry.h"
+#include "flutter/shell/platform/embedder/embedder.h"
+
+namespace flutter {
+
+class TextInputPluginDelegate {
+ public:
+  // Notifies delegate that the cursor position has changed.
+  virtual void OnCursorRectUpdated(const Rect& rect) = 0;
+};
+
+}  // namespace flutter
+
+#endif  // FLUTTER_SHELL_PLATFORM_WINDOWS_TEXT_INPUT_PLUGIN_DELEGATE_H_

shell/platform/windows/win32_flutter_window.cc

@@ -178,4 +178,8 @@ void Win32FlutterWindow::OnScroll(double delta_x, double delta_y) {
                                       kScrollOffsetMultiplier);
 }
 
+void Win32FlutterWindow::UpdateCursorRect(const Rect& rect) {
+  // TODO(cbracken): Implement IMM candidate window positioning.
+}
+
 }  // namespace flutter

shell/platform/windows/win32_flutter_window.h

@@ -11,6 +11,7 @@
 #include <string>
 #include <vector>
 
+#include "flutter/shell/platform/common/cpp/geometry.h"
 #include "flutter/shell/platform/embedder/embedder.h"
 #include "flutter/shell/platform/windows/flutter_windows_view.h"
 #include "flutter/shell/platform/windows/win32_window.h"
@@ -74,6 +75,9 @@ class Win32FlutterWindow : public Win32Window, public WindowBindingHandler {
   // |FlutterWindowBindingHandler|
   void UpdateFlutterCursor(const std::string& cursor_name) override;
 
+  // |FlutterWindowBindingHandler|
+  void UpdateCursorRect(const Rect& rect) override;
+
   // |FlutterWindowBindingHandler|
   void OnWindowResized() override;
 
@@ -84,6 +88,9 @@ class Win32FlutterWindow : public Win32Window, public WindowBindingHandler {
 
   // The last cursor set by Flutter. Defaults to the arrow cursor.
   HCURSOR current_cursor_;
+
+  // The cursor rect set by Flutter.
+  RECT cursor_rect_;
 };
 
 }  // namespace flutter

shell/platform/windows/window_binding_handler.h

@@ -10,6 +10,7 @@
 #include <string>
 #include <variant>
 
+#include "flutter/shell/platform/common/cpp/geometry.h"
 #include "flutter/shell/platform/windows/public/flutter_windows.h"
 #include "flutter/shell/platform/windows/window_binding_handler_delegate.h"
 
@@ -51,6 +52,9 @@ class WindowBindingHandler {
   // Sets the cursor that should be used when the mouse is over the Flutter
   // content. See mouse_cursor.dart for the values and meanings of cursor_name.
   virtual void UpdateFlutterCursor(const std::string& cursor_name) = 0;
+
+  // Sets the cursor rect in root view coordinates.
+  virtual void UpdateCursorRect(const Rect& rect) = 0;
 };
 
 }  // namespace flutter

shell/platform/windows/window_binding_handler_delegate.h

@@ -5,6 +5,7 @@
 #ifndef FLUTTER_SHELL_PLATFORM_WINDOWS_WINDOW_BINDING_HANDLER_DELEGATE_H_
 #define FLUTTER_SHELL_PLATFORM_WINDOWS_WINDOW_BINDING_HANDLER_DELEGATE_H_
 
+#include "flutter/shell/platform/common/cpp/geometry.h"
 #include "flutter/shell/platform/embedder/embedder.h"
 
 namespace flutter {