Improve Unicode handling on Windows (#11899)

Significantly improves the behavior of non-ASCII text input on Windows. Correctly
processes incoming character events as UTF-16, and for now uses UTF-32 for
the text model so that the existing index-based logic will work much more often.

Future work is still needed, but this will handle far more cases correctly.

shell/platform/common/cpp/text_input_model.cc

@@ -4,7 +4,9 @@
 
 #include "flutter/shell/platform/common/cpp/text_input_model.h"
 
+#include <codecvt>
 #include <iostream>
+#include <locale>
 
 // TODO(awdavies): Need to fix this regarding issue #47.
 static constexpr char kComposingBaseKey[] = "composingBase";
@@ -26,11 +28,16 @@ static constexpr char kTextInputAction[] = "inputAction";
 static constexpr char kTextInputType[] = "inputType";
 static constexpr char kTextInputTypeName[] = "name";
 
+#if defined(_MSC_VER)
+// TODO(naifu): This temporary code is to solve link error.(VS2015/2017)
+// https://social.msdn.microsoft.com/Forums/vstudio/en-US/8f40dcd8-c67f-4eba-9134-a19b9178e481/vs-2015-rc-linker-stdcodecvt-error
+std::locale::id std::codecvt<char32_t, char, _Mbstatet>::id;
+#endif  // defined(_MSC_VER)
+
 namespace flutter {
 
 TextInputModel::TextInputModel(int client_id, const rapidjson::Value& config)
-    : text_(""),
-      client_id_(client_id),
+    : client_id_(client_id),
       selection_base_(text_.begin()),
       selection_extent_(text_.begin()) {
   // TODO: Improve error handling during refactoring; this is just minimal
@@ -64,7 +71,8 @@ bool TextInputModel::SetEditingState(size_t selection_base,
   if (selection_extent > text.size()) {
     return false;
   }
-  text_ = std::string(text);
+  std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf32conv;
+  text_ = utf32conv.from_bytes(text);
   selection_base_ = text_.begin() + selection_base;
   selection_extent_ = text_.begin() + selection_extent;
   return true;
@@ -76,7 +84,7 @@ void TextInputModel::DeleteSelected() {
   selection_extent_ = selection_base_;
 }
 
-void TextInputModel::AddCharacter(char c) {
+void TextInputModel::AddCharacter(char32_t c) {
   if (selection_base_ != selection_extent_) {
     DeleteSelected();
   }
@@ -172,8 +180,10 @@ std::unique_ptr<rapidjson::Document> TextInputModel::GetState() const {
                           static_cast<int>(selection_extent_ - text_.begin()),
                           allocator);
   editing_state.AddMember(kSelectionIsDirectionalKey, false, allocator);
-  editing_state.AddMember(kTextKey, rapidjson::Value(text_, allocator).Move(),
-                          allocator);
+  std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;
+  editing_state.AddMember(
+      kTextKey, rapidjson::Value(utf8conv.to_bytes(text_), allocator).Move(),
+      allocator);
   args->PushBack(editing_state, allocator);
   return args;
 }

shell/platform/common/cpp/text_input_model.h

@@ -32,7 +32,7 @@ class TextInputModel {
   // Either appends after the cursor (when selection base and extent are the
   // same), or deletes the selected characters, replacing the text with the
   // character specified.
-  void AddCharacter(char c);
+  void AddCharacter(char32_t c);
 
   // Deletes either the selection, or one character ahead of the cursor.
   //
@@ -89,12 +89,12 @@ class TextInputModel {
  private:
   void DeleteSelected();
 
-  std::string text_;
+  std::u32string text_;
   int client_id_;
   std::string input_type_;
   std::string input_action_;
-  std::string::iterator selection_base_;
-  std::string::iterator selection_extent_;
+  std::u32string::iterator selection_base_;
+  std::u32string::iterator selection_extent_;
 };
 
 }  // namespace flutter

shell/platform/glfw/text_input_plugin.cc

@@ -40,9 +40,7 @@ void TextInputPlugin::CharHook(GLFWwindow* window, unsigned int code_point) {
   if (active_model_ == nullptr) {
     return;
   }
-  // TODO(awdavies): Actually handle potential unicode characters. Probably
-  // requires some ICU data or something.
-  active_model_->AddCharacter(static_cast<char>(code_point));
+  active_model_->AddCharacter(code_point);
   SendStateUpdate(*active_model_);
 }
 

shell/platform/windows/key_event_handler.cc

@@ -32,7 +32,7 @@ KeyEventHandler::KeyEventHandler(flutter::BinaryMessenger* messenger)
 KeyEventHandler::~KeyEventHandler() = default;
 
 void KeyEventHandler::CharHook(Win32FlutterWindow* window,
-                               unsigned int code_point) {}
+                               char32_t code_point) {}
 
 void KeyEventHandler::KeyboardHook(Win32FlutterWindow* window,
                                    int key,

shell/platform/windows/key_event_handler.h

@@ -34,7 +34,7 @@ class KeyEventHandler : public KeyboardHookHandler {
                     int mods) override;
 
   // |KeyboardHookHandler|
-  void CharHook(Win32FlutterWindow* window, unsigned int code_point) override;
+  void CharHook(Win32FlutterWindow* window, char32_t code_point) override;
 
  private:
   // The Flutter system channel for key event messages.

shell/platform/windows/keyboard_hook_handler.h

@@ -24,8 +24,7 @@ class KeyboardHookHandler {
                             int mods) = 0;
 
   // A function for hooking into unicode code point input.
-  virtual void CharHook(Win32FlutterWindow* window,
-                        unsigned int code_point) = 0;
+  virtual void CharHook(Win32FlutterWindow* window, char32_t code_point) = 0;
 };
 
 }  // namespace flutter

shell/platform/windows/text_input_plugin.cc

@@ -39,12 +39,11 @@ static constexpr uint32_t kInputModelLimit = 256;
 namespace flutter {
 
 void TextInputPlugin::CharHook(Win32FlutterWindow* window,
-                               unsigned int code_point) {
+                               char32_t code_point) {
   if (active_model_ == nullptr) {
     return;
   }
-  // TODO bug 30661
-  active_model_->AddCharacter(static_cast<char>(code_point));
+  active_model_->AddCharacter(code_point);
   SendStateUpdate(*active_model_);
 }
 

shell/platform/windows/text_input_plugin.h

@@ -35,7 +35,7 @@ class TextInputPlugin : public KeyboardHookHandler {
                     int mods) override;
 
   // |KeyboardHookHandler|
-  void CharHook(Win32FlutterWindow* window, unsigned int code_point) override;
+  void CharHook(Win32FlutterWindow* window, char32_t code_point) override;
 
  private:
   // Sends the current state of the given model to the Flutter engine.

shell/platform/windows/win32_flutter_window.cc

@@ -125,7 +125,7 @@ void Win32FlutterWindow::OnPointerUp(double x, double y) {
   }
 }
 
-void Win32FlutterWindow::OnChar(unsigned int code_point) {
+void Win32FlutterWindow::OnChar(char32_t code_point) {
   if (process_events_) {
     SendChar(code_point);
   }
@@ -207,7 +207,7 @@ void Win32FlutterWindow::SendPointerUp(double x, double y) {
   SendPointerEventWithData(event);
 }
 
-void Win32FlutterWindow::SendChar(unsigned int code_point) {
+void Win32FlutterWindow::SendChar(char32_t code_point) {
   for (const auto& handler : keyboard_hook_handlers_) {
     handler->CharHook(this, code_point);
   }

shell/platform/windows/win32_flutter_window.h

@@ -54,7 +54,7 @@ class Win32FlutterWindow : public Win32Window {
   void OnPointerUp(double x, double y) override;
 
   // |Win32Window|
-  void OnChar(unsigned int code_point) override;
+  void OnChar(char32_t code_point) override;
 
   // |Win32Window|
   void OnKey(int key, int scancode, int action, int mods) override;
@@ -103,7 +103,7 @@ class Win32FlutterWindow : public Win32Window {
   void SendPointerUp(double x, double y);
 
   // Reports a keyboard character to Flutter engine.
-  void SendChar(unsigned int code_point);
+  void SendChar(char32_t code_point);
 
   // Reports a raw keyboard message to Flutter engine.
   void SendKey(int key, int scancode, int action, int mods);

shell/platform/windows/win32_window.cc

@@ -153,13 +153,33 @@ Win32Window::MessageHandler(HWND hwnd,
         window->OnScroll(
             0.0, -(static_cast<short>(HIWORD(wparam)) / (double)WHEEL_DELTA));
         break;
+      case WM_UNICHAR: {
+        // Tell third-pary app, we can support Unicode.
+        if (wparam == UNICODE_NOCHAR)
+          return TRUE;
+        // DefWindowProc will send WM_CHAR for this WM_UNICHAR.
+        break;
+      }
       case WM_CHAR:
-      case WM_SYSCHAR:
-      case WM_UNICHAR:
-        if (wparam != VK_BACK) {
-          window->OnChar(static_cast<unsigned int>(wparam));
+      case WM_SYSCHAR: {
+        if (wparam == VK_BACK)
+          break;
+        char32_t code_point = static_cast<char32_t>(wparam);
+        static char32_t lead_surrogate = 0;
+        // If code_point is LeadSurrogate, save and return.
+        if ((code_point & 0xFFFFFC00) == 0xD800) {
+          lead_surrogate = code_point;
+          return TRUE;
+        }
+        // Merge TrailSurrogate and LeadSurrogate.
+        if (lead_surrogate != 0 && (code_point & 0xFFFFFC00) == 0xDC00) {
+          code_point = 0x10000 + ((lead_surrogate & 0x000003FF) << 10) +
+                       (code_point & 0x3FF);
         }
+        lead_surrogate = 0;
+        window->OnChar(code_point);
         break;
+      }
       case WM_KEYDOWN:
       case WM_SYSKEYDOWN:
       case WM_KEYUP:

shell/platform/windows/win32_window.h

@@ -89,7 +89,7 @@ class Win32Window {
   virtual void OnPointerUp(double x, double y) = 0;
 
   // Called when character input occurs.
-  virtual void OnChar(unsigned int code_point) = 0;
+  virtual void OnChar(char32_t code_point) = 0;
 
   // Called when raw keyboard input occurs.
   virtual void OnKey(int key, int scancode, int action, int mods) = 0;