Distinguish between touch and mouse input on win32/winuwp (#25579)

shell/platform/embedder/embedder.cc

@@ -1390,6 +1390,8 @@ inline flutter::PointerData::DeviceKind ToPointerDataKind(
       return flutter::PointerData::DeviceKind::kMouse;
     case kFlutterPointerDeviceKindTouch:
       return flutter::PointerData::DeviceKind::kTouch;
+    case kFlutterPointerDeviceKindStylus:
+      return flutter::PointerData::DeviceKind::kStylus;
   }
   return flutter::PointerData::DeviceKind::kMouse;
 }

shell/platform/embedder/embedder.h

@@ -603,6 +603,7 @@ typedef enum {
 typedef enum {
   kFlutterPointerDeviceKindMouse = 1,
   kFlutterPointerDeviceKindTouch,
+  kFlutterPointerDeviceKindStylus,
 } FlutterPointerDeviceKind;
 
 /// Flags for the `buttons` field of `FlutterPointerEvent` when `device_kind`

shell/platform/windows/flutter_window_win32.cc

@@ -120,6 +120,24 @@ static uint64_t ConvertWinButtonToFlutterButton(UINT button) {
   return 0;
 }
 
+// This method is only valid during a window message related to mouse/touch
+// input.
+// See
+// https://docs.microsoft.com/en-us/windows/win32/tablet/system-events-and-mouse-messages?redirectedfrom=MSDN#distinguishing-pen-input-from-mouse-and-touch.
+static FlutterPointerDeviceKind GetFlutterPointerDeviceKind() {
+  constexpr LPARAM kTouchOrPenSignature = 0xFF515700;
+  constexpr LPARAM kTouchSignature = kTouchOrPenSignature | 0x80;
+  constexpr LPARAM kSignatureMask = 0xFFFFFF00;
+  LPARAM info = GetMessageExtraInfo();
+  if ((info & kSignatureMask) == kTouchOrPenSignature) {
+    if ((info & kTouchSignature) == kTouchSignature) {
+      return kFlutterPointerDeviceKindTouch;
+    }
+    return kFlutterPointerDeviceKindStylus;
+  }
+  return kFlutterPointerDeviceKindMouse;
+}
+
 void FlutterWindowWin32::OnDpiScale(unsigned int dpi){};
 
 // When DesktopWindow notifies that a WM_Size message has come in
@@ -131,14 +149,15 @@ void FlutterWindowWin32::OnResize(unsigned int width, unsigned int height) {
 }
 
 void FlutterWindowWin32::OnPointerMove(double x, double y) {
-  binding_handler_delegate_->OnPointerMove(x, y);
+  binding_handler_delegate_->OnPointerMove(x, y, GetFlutterPointerDeviceKind());
 }
 
 void FlutterWindowWin32::OnPointerDown(double x, double y, UINT button) {
   uint64_t flutter_button = ConvertWinButtonToFlutterButton(button);
   if (flutter_button != 0) {
     binding_handler_delegate_->OnPointerDown(
-        x, y, static_cast<FlutterPointerMouseButtons>(flutter_button));
+        x, y, GetFlutterPointerDeviceKind(),
+        static_cast<FlutterPointerMouseButtons>(flutter_button));
   }
 }
 
@@ -146,12 +165,13 @@ void FlutterWindowWin32::OnPointerUp(double x, double y, UINT button) {
   uint64_t flutter_button = ConvertWinButtonToFlutterButton(button);
   if (flutter_button != 0) {
     binding_handler_delegate_->OnPointerUp(
-        x, y, static_cast<FlutterPointerMouseButtons>(flutter_button));
+        x, y, GetFlutterPointerDeviceKind(),
+        static_cast<FlutterPointerMouseButtons>(flutter_button));
   }
 }
 
 void FlutterWindowWin32::OnPointerLeave() {
-  binding_handler_delegate_->OnPointerLeave();
+  binding_handler_delegate_->OnPointerLeave(GetFlutterPointerDeviceKind());
 }
 
 void FlutterWindowWin32::OnSetCursor() {

shell/platform/windows/flutter_window_win32_unittests.cc

@@ -159,6 +159,38 @@ class MockFlutterWindowWin32 : public FlutterWindowWin32 {
   MOCK_METHOD1(UpdateCursorRect, void(const Rect&));
 };
 
+class MockWindowBindingHandlerDelegate : public WindowBindingHandlerDelegate {
+ public:
+  MockWindowBindingHandlerDelegate() {}
+
+  // Prevent copying.
+  MockWindowBindingHandlerDelegate(MockWindowBindingHandlerDelegate const&) =
+      delete;
+  MockWindowBindingHandlerDelegate& operator=(
+      MockWindowBindingHandlerDelegate const&) = delete;
+
+  MOCK_METHOD2(OnWindowSizeChanged, void(size_t, size_t));
+  MOCK_METHOD3(OnPointerMove, void(double, double, FlutterPointerDeviceKind));
+  MOCK_METHOD4(OnPointerDown,
+               void(double,
+                    double,
+                    FlutterPointerDeviceKind,
+                    FlutterPointerMouseButtons));
+  MOCK_METHOD4(OnPointerUp,
+               void(double,
+                    double,
+                    FlutterPointerDeviceKind,
+                    FlutterPointerMouseButtons));
+  MOCK_METHOD1(OnPointerLeave, void(FlutterPointerDeviceKind));
+  MOCK_METHOD1(OnText, void(const std::u16string&));
+  MOCK_METHOD6(OnKey, bool(int, int, int, char32_t, bool, bool));
+  MOCK_METHOD0(OnComposeBegin, void());
+  MOCK_METHOD0(OnComposeCommit, void());
+  MOCK_METHOD0(OnComposeEnd, void());
+  MOCK_METHOD2(OnComposeChange, void(const std::u16string&, int));
+  MOCK_METHOD5(OnScroll, void(double, double, double, double, int));
+};
+
 // A FlutterWindowsView that overrides the RegisterKeyboardHandlers function
 // to register the keyboard hook handlers that can be spied upon.
 class TestFlutterWindowsView : public FlutterWindowsView {
@@ -509,5 +541,76 @@ TEST(FlutterWindowWin32Test, OnCursorRectUpdatedHighDPI) {
   win32window.OnCursorRectUpdated(cursor_rect);
 }
 
+TEST(FlutterWindowWin32Test, OnPointerStarSendsDeviceType) {
+  FlutterWindowWin32 win32window(100, 100);
+  MockWindowBindingHandlerDelegate delegate;
+  win32window.SetView(&delegate);
+  // Move
+  EXPECT_CALL(delegate,
+              OnPointerMove(10.0, 10.0, kFlutterPointerDeviceKindMouse))
+      .Times(1);
+  EXPECT_CALL(delegate,
+              OnPointerMove(10.0, 10.0, kFlutterPointerDeviceKindTouch))
+      .Times(1);
+  EXPECT_CALL(delegate,
+              OnPointerMove(10.0, 10.0, kFlutterPointerDeviceKindStylus))
+      .Times(1);
+
+  // Down
+  EXPECT_CALL(delegate,
+              OnPointerDown(10.0, 10.0, kFlutterPointerDeviceKindMouse,
+                            kFlutterPointerButtonMousePrimary))
+      .Times(1);
+  EXPECT_CALL(delegate,
+              OnPointerDown(10.0, 10.0, kFlutterPointerDeviceKindTouch,
+                            kFlutterPointerButtonMousePrimary))
+      .Times(1);
+  EXPECT_CALL(delegate,
+              OnPointerDown(10.0, 10.0, kFlutterPointerDeviceKindStylus,
+                            kFlutterPointerButtonMousePrimary))
+      .Times(1);
+
+  // Up
+  EXPECT_CALL(delegate, OnPointerUp(10.0, 10.0, kFlutterPointerDeviceKindMouse,
+                                    kFlutterPointerButtonMousePrimary))
+      .Times(1);
+  EXPECT_CALL(delegate, OnPointerUp(10.0, 10.0, kFlutterPointerDeviceKindTouch,
+                                    kFlutterPointerButtonMousePrimary))
+      .Times(1);
+  EXPECT_CALL(delegate, OnPointerUp(10.0, 10.0, kFlutterPointerDeviceKindStylus,
+                                    kFlutterPointerButtonMousePrimary))
+      .Times(1);
+
+  // Leave
+  EXPECT_CALL(delegate, OnPointerLeave(kFlutterPointerDeviceKindMouse))
+      .Times(1);
+  EXPECT_CALL(delegate, OnPointerLeave(kFlutterPointerDeviceKindTouch))
+      .Times(1);
+  EXPECT_CALL(delegate, OnPointerLeave(kFlutterPointerDeviceKindStylus))
+      .Times(1);
+
+  win32window.OnPointerMove(10.0, 10.0);
+  win32window.OnPointerDown(10.0, 10.0, WM_LBUTTONDOWN);
+  win32window.OnPointerUp(10.0, 10.0, WM_LBUTTONDOWN);
+  win32window.OnPointerLeave();
+
+  // Touch
+  LPARAM original_lparam = SetMessageExtraInfo(0xFF51578b);
+  win32window.OnPointerMove(10.0, 10.0);
+  win32window.OnPointerDown(10.0, 10.0, WM_LBUTTONDOWN);
+  win32window.OnPointerUp(10.0, 10.0, WM_LBUTTONDOWN);
+  win32window.OnPointerLeave();
+
+  // Pen
+  SetMessageExtraInfo(0xFF515700);
+  win32window.OnPointerMove(10.0, 10.0);
+  win32window.OnPointerDown(10.0, 10.0, WM_LBUTTONDOWN);
+  win32window.OnPointerUp(10.0, 10.0, WM_LBUTTONDOWN);
+  win32window.OnPointerLeave();
+
+  // Reset extra info for other tests.
+  SetMessageExtraInfo(original_lparam);
+}
+
 }  // namespace testing
 }  // namespace flutter

shell/platform/windows/flutter_window_winuwp.cc

@@ -126,9 +126,11 @@ void FlutterWindowWinUWP::OnPointerPressed(
     winrt::Windows::UI::Core::PointerEventArgs const& args) {
   double x = GetPosX(args);
   double y = GetPosY(args);
+  FlutterPointerDeviceKind device_kind = GetPointerDeviceKind(args);
 
   binding_handler_delegate_->OnPointerDown(
-      x, y, FlutterPointerMouseButtons::kFlutterPointerButtonMousePrimary);
+      x, y, device_kind,
+      FlutterPointerMouseButtons::kFlutterPointerButtonMousePrimary);
 }
 
 void FlutterWindowWinUWP::OnPointerReleased(
@@ -136,9 +138,11 @@ void FlutterWindowWinUWP::OnPointerReleased(
     winrt::Windows::UI::Core::PointerEventArgs const& args) {
   double x = GetPosX(args);
   double y = GetPosY(args);
+  FlutterPointerDeviceKind device_kind = GetPointerDeviceKind(args);
 
   binding_handler_delegate_->OnPointerUp(
-      x, y, FlutterPointerMouseButtons::kFlutterPointerButtonMousePrimary);
+      x, y, device_kind,
+      FlutterPointerMouseButtons::kFlutterPointerButtonMousePrimary);
 }
 
 void FlutterWindowWinUWP::OnPointerMoved(
@@ -146,8 +150,9 @@ void FlutterWindowWinUWP::OnPointerMoved(
     winrt::Windows::UI::Core::PointerEventArgs const& args) {
   double x = GetPosX(args);
   double y = GetPosY(args);
+  FlutterPointerDeviceKind device_kind = GetPointerDeviceKind(args);
 
-  binding_handler_delegate_->OnPointerMove(x, y);
+  binding_handler_delegate_->OnPointerMove(x, y, device_kind);
 }
 
 void FlutterWindowWinUWP::OnPointerWheelChanged(
@@ -176,6 +181,19 @@ double FlutterWindowWinUWP::GetPosY(
                              inverse_dpi_scale);
 }
 
+FlutterPointerDeviceKind FlutterWindowWinUWP::GetPointerDeviceKind(
+    winrt::Windows::UI::Core::PointerEventArgs const& args) {
+  switch (args.CurrentPoint().PointerDevice().PointerDeviceType()) {
+    case winrt::Windows::Devices::Input::PointerDeviceType::Mouse:
+      return kFlutterPointerDeviceKindMouse;
+    case winrt::Windows::Devices::Input::PointerDeviceType::Pen:
+      return kFlutterPointerDeviceKindStylus;
+    case winrt::Windows::Devices::Input::PointerDeviceType::Touch:
+      return kFlutterPointerDeviceKindTouch;
+  }
+  return kFlutterPointerDeviceKindMouse;
+}
+
 void FlutterWindowWinUWP::OnBoundsChanged(
     winrt::Windows::UI::ViewManagement::ApplicationView const& app_view,
     winrt::Windows::Foundation::IInspectable const&) {

shell/platform/windows/flutter_window_winuwp.h

@@ -5,6 +5,7 @@
 #ifndef FLUTTER_SHELL_PLATFORM_WINDOWS_UWP_FLUTTER_WINDOW_H_
 #define FLUTTER_SHELL_PLATFORM_WINDOWS_UWP_FLUTTER_WINDOW_H_
 
+#include <winrt/Windows.Devices.Input.h>
 #include <winrt/Windows.UI.Composition.h>
 #include <winrt/Windows.UI.Input.h>
 #include <winrt/Windows.UI.Text.Core.h>
@@ -111,6 +112,10 @@ class FlutterWindowWinUWP : public WindowBindingHandler {
   // Converts from logical point to physical Y value.
   double GetPosY(winrt::Windows::UI::Core::PointerEventArgs const& args);
 
+  // Gets the pointer kind.
+  FlutterPointerDeviceKind GetPointerDeviceKind(
+      winrt::Windows::UI::Core::PointerEventArgs const& args);
+
   // Backing CoreWindow. nullptr if not set.
   winrt::Windows::UI::Core::CoreWindow window_{nullptr};
 

shell/platform/windows/flutter_windows_view.cc

@@ -167,34 +167,38 @@ void FlutterWindowsView::OnWindowSizeChanged(size_t width, size_t height) {
   }
 }
 
-void FlutterWindowsView::OnPointerMove(double x, double y) {
-  SendPointerMove(x, y);
+void FlutterWindowsView::OnPointerMove(double x,
+                                       double y,
+                                       FlutterPointerDeviceKind device_kind) {
+  SendPointerMove(x, y, device_kind);
 }
 
 void FlutterWindowsView::OnPointerDown(
     double x,
     double y,
+    FlutterPointerDeviceKind device_kind,
     FlutterPointerMouseButtons flutter_button) {
   if (flutter_button != 0) {
     uint64_t mouse_buttons = mouse_state_.buttons | flutter_button;
     SetMouseButtons(mouse_buttons);
-    SendPointerDown(x, y);
+    SendPointerDown(x, y, device_kind);
   }
 }
 
 void FlutterWindowsView::OnPointerUp(
     double x,
     double y,
+    FlutterPointerDeviceKind device_kind,
     FlutterPointerMouseButtons flutter_button) {
   if (flutter_button != 0) {
     uint64_t mouse_buttons = mouse_state_.buttons & ~flutter_button;
     SetMouseButtons(mouse_buttons);
-    SendPointerUp(x, y);
+    SendPointerUp(x, y, device_kind);
   }
 }
 
-void FlutterWindowsView::OnPointerLeave() {
-  SendPointerLeave();
+void FlutterWindowsView::OnPointerLeave(FlutterPointerDeviceKind device_kind) {
+  SendPointerLeave(device_kind);
 }
 
 void FlutterWindowsView::OnText(const std::u16string& text) {
@@ -275,36 +279,47 @@ void FlutterWindowsView::SetEventPhaseFromCursorButtonState(
   }
 }
 
-void FlutterWindowsView::SendPointerMove(double x, double y) {
+void FlutterWindowsView::SendPointerMove(double x,
+                                         double y,
+                                         FlutterPointerDeviceKind device_kind) {
   FlutterPointerEvent event = {};
   event.x = x;
   event.y = y;
+  event.device_kind = device_kind;
   SetEventPhaseFromCursorButtonState(&event);
   SendPointerEventWithData(event);
 }
 
-void FlutterWindowsView::SendPointerDown(double x, double y) {
+void FlutterWindowsView::SendPointerDown(double x,
+                                         double y,
+                                         FlutterPointerDeviceKind device_kind) {
   FlutterPointerEvent event = {};
   SetEventPhaseFromCursorButtonState(&event);
   event.x = x;
   event.y = y;
+  event.device_kind = device_kind;
   SendPointerEventWithData(event);
   SetMouseFlutterStateDown(true);
 }
 
-void FlutterWindowsView::SendPointerUp(double x, double y) {
+void FlutterWindowsView::SendPointerUp(double x,
+                                       double y,
+                                       FlutterPointerDeviceKind device_kind) {
   FlutterPointerEvent event = {};
   SetEventPhaseFromCursorButtonState(&event);
   event.x = x;
   event.y = y;
+  event.device_kind = device_kind;
   SendPointerEventWithData(event);
   if (event.phase == FlutterPointerPhase::kUp) {
     SetMouseFlutterStateDown(false);
   }
 }
 
-void FlutterWindowsView::SendPointerLeave() {
+void FlutterWindowsView::SendPointerLeave(
+    FlutterPointerDeviceKind device_kind) {
   FlutterPointerEvent event = {};
+  event.device_kind = device_kind;
   event.phase = FlutterPointerPhase::kRemove;
   SendPointerEventWithData(event);
 }
@@ -392,7 +407,6 @@ void FlutterWindowsView::SendPointerEventWithData(
   }
 
   FlutterPointerEvent event = event_data;
-  event.device_kind = kFlutterPointerDeviceKindMouse;
   event.buttons = mouse_state_.buttons;
 
   // Set metadata that's always the same regardless of the event.

shell/platform/windows/flutter_windows_view.h

@@ -87,20 +87,24 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate,
   void OnWindowSizeChanged(size_t width, size_t height) override;
 
   // |WindowBindingHandlerDelegate|
-  void OnPointerMove(double x, double y) override;
+  void OnPointerMove(double x,
+                     double y,
+                     FlutterPointerDeviceKind device_kind) override;
 
   // |WindowBindingHandlerDelegate|
   void OnPointerDown(double x,
                      double y,
+                     FlutterPointerDeviceKind device_kind,
                      FlutterPointerMouseButtons button) override;
 
   // |WindowBindingHandlerDelegate|
   void OnPointerUp(double x,
                    double y,
+                   FlutterPointerDeviceKind device_kind,
                    FlutterPointerMouseButtons button) override;
 
   // |WindowBindingHandlerDelegate|
-  void OnPointerLeave() override;
+  void OnPointerLeave(FlutterPointerDeviceKind device_kind) override;
 
   // |WindowBindingHandlerDelegate|
   void OnText(const std::u16string&) override;
@@ -178,20 +182,24 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate,
   void SendWindowMetrics(size_t width, size_t height, double dpiscale) const;
 
   // Reports a mouse movement to Flutter engine.
-  void SendPointerMove(double x, double y);
+  void SendPointerMove(double x,
+                       double y,
+                       FlutterPointerDeviceKind device_kind);
 
   // Reports mouse press to Flutter engine.
-  void SendPointerDown(double x, double y);
+  void SendPointerDown(double x,
+                       double y,
+                       FlutterPointerDeviceKind device_kind);
 
   // Reports mouse release to Flutter engine.
-  void SendPointerUp(double x, double y);
+  void SendPointerUp(double x, double y, FlutterPointerDeviceKind device_kind);
 
   // Reports mouse left the window client area.
   //
   // Win32 api doesn't have "mouse enter" event. Therefore, there is no
   // SendPointerEnter method. A mouse enter event is tracked then the "move"
   // event is called.
-  void SendPointerLeave();
+  void SendPointerLeave(FlutterPointerDeviceKind device_kind);
 
   // Reports a keyboard character to Flutter engine.
   void SendText(const std::u16string&);

shell/platform/windows/game_pad_cursor_winuwp.cc

@@ -159,7 +159,10 @@ void GamepadCursorWinUWP::OnGamepadLeftStickMoved(double x, double y) {
 
     winrt::Windows::Foundation::Numerics::float3 scaled =
         GetScaledInput(cursor_visual_.Offset());
-    binding_handler_delegate_->OnPointerMove(scaled.x, scaled.y);
+    // TODO(dnfield): Support for gamepad as a distinct device type?
+    // https://github.com/flutter/flutter/issues/80472
+    binding_handler_delegate_->OnPointerMove(scaled.x, scaled.y,
+                                             kFlutterPointerDeviceKindMouse);
   }
 }
 
@@ -183,8 +186,10 @@ void GamepadCursorWinUWP::OnGamepadButtonPressed(
     // handling defered delivery, remove the need for action value.
     // https://github.com/flutter/flutter/issues/70202
 
+    // TODO(dnfield): Support for gamepad as a distinct device type?
+    // https://github.com/flutter/flutter/issues/80472
     binding_handler_delegate_->OnPointerDown(
-        scaled.x, scaled.y,
+        scaled.x, scaled.y, kFlutterPointerDeviceKindMouse,
         FlutterPointerMouseButtons::kFlutterPointerButtonMousePrimary);
   } else if ((buttons &
               winrt::Windows::Gaming::Input::GamepadButtons::DPadLeft) ==
@@ -225,8 +230,10 @@ void GamepadCursorWinUWP::OnGamepadButtonReleased(
     // handling defered delivery, remove the need for action value.
     // https://github.com/flutter/flutter/issues/70202
 
+    // TODO(dnfield): Support for gamepad as a distinct device type?
+    // https://github.com/flutter/flutter/issues/80472
     binding_handler_delegate_->OnPointerUp(
-        scaled.x, scaled.y,
+        scaled.x, scaled.y, kFlutterPointerDeviceKindMouse,
         FlutterPointerMouseButtons::kFlutterPointerButtonMousePrimary);
   } else if ((buttons &
               winrt::Windows::Gaming::Input::GamepadButtons::DPadLeft) ==

shell/platform/windows/window_binding_handler_delegate.h

@@ -19,23 +19,27 @@ class WindowBindingHandlerDelegate {
 
   // Notifies delegate that backing window mouse has moved.
   // Typically called by currently configured WindowBindingHandler
-  virtual void OnPointerMove(double x, double y) = 0;
+  virtual void OnPointerMove(double x,
+                             double y,
+                             FlutterPointerDeviceKind device_kind) = 0;
 
   // Notifies delegate that backing window mouse pointer button has been
   // pressed. Typically called by currently configured WindowBindingHandler
   virtual void OnPointerDown(double x,
                              double y,
+                             FlutterPointerDeviceKind device_kind,
                              FlutterPointerMouseButtons button) = 0;
 
   // Notifies delegate that backing window mouse pointer button has been
   // released. Typically called by currently configured WindowBindingHandler
   virtual void OnPointerUp(double x,
                            double y,
+                           FlutterPointerDeviceKind device_kind,
                            FlutterPointerMouseButtons button) = 0;
 
   // Notifies delegate that backing window mouse pointer has left the window.
   // Typically called by currently configured WindowBindingHandler
-  virtual void OnPointerLeave() = 0;
+  virtual void OnPointerLeave(FlutterPointerDeviceKind device_kind) = 0;
 
   // Notifies delegate that backing window has received text.
   // Typically called by currently configured WindowBindingHandler