Implement delayed key event synthesis for Windows

ci/licenses_golden/licenses_flutter

@@ -1475,6 +1475,7 @@ FILE: ../../../flutter/shell/platform/windows/flutter_windows_win32.cc
 FILE: ../../../flutter/shell/platform/windows/flutter_windows_winuwp.cc
 FILE: ../../../flutter/shell/platform/windows/key_event_handler.cc
 FILE: ../../../flutter/shell/platform/windows/key_event_handler.h
+FILE: ../../../flutter/shell/platform/windows/key_event_handler_unittests.cc
 FILE: ../../../flutter/shell/platform/windows/keyboard_hook_handler.h
 FILE: ../../../flutter/shell/platform/windows/platform_handler.cc
 FILE: ../../../flutter/shell/platform/windows/platform_handler.h

shell/platform/windows/BUILD.gn

@@ -167,11 +167,14 @@ if (target_os == "winuwp") {
       "flutter_project_bundle_unittests.cc",
       "flutter_windows_engine_unittests.cc",
       "flutter_windows_texture_registrar_unittests.cc",
+      "key_event_handler_unittests.cc",
       "string_conversion_unittests.cc",
       "system_utils_unittests.cc",
       "testing/engine_embedder_api_modifier.h",
       "testing/mock_win32_window.cc",
       "testing/mock_win32_window.h",
+      "testing/mock_window_binding_handler.cc",
+      "testing/mock_window_binding_handler.h",
       "testing/win32_flutter_window_test.cc",
       "testing/win32_flutter_window_test.h",
       "win32_dpi_utils_unittests.cc",

shell/platform/windows/flutter_windows_engine_unittests.cc

@@ -102,7 +102,7 @@ TEST(FlutterWindowsEngine, SendPlatformMessageWithoutResponse) {
   const char* channel = "test";
   const std::vector<uint8_t> test_message = {1, 2, 3, 4};
 
-  // Without a respones, SendPlatformMessage should be a simple passthrough.
+  // Without a response, SendPlatformMessage should be a simple pass-through.
   bool called = false;
   modifier.embedder_api().SendPlatformMessage = MOCK_ENGINE_PROC(
       SendPlatformMessage, ([&called, test_message](auto engine, auto message) {

shell/platform/windows/flutter_windows_view.cc

@@ -35,10 +35,7 @@ void FlutterWindowsView::SetEngine(
 
   // Set up the system channel handlers.
   auto internal_plugin_messenger = internal_plugin_registrar_->messenger();
-  keyboard_hook_handlers_.push_back(
-      std::make_unique<flutter::KeyEventHandler>(internal_plugin_messenger));
-  keyboard_hook_handlers_.push_back(std::make_unique<flutter::TextInputPlugin>(
-      internal_plugin_messenger, this));
+  RegisterKeyboardHookHandlers(internal_plugin_messenger);
   platform_handler_ = PlatformHandler::Create(internal_plugin_messenger, this);
   cursor_handler_ = std::make_unique<flutter::CursorHandler>(
       internal_plugin_messenger, binding_handler_.get());
@@ -49,6 +46,18 @@ void FlutterWindowsView::SetEngine(
                     binding_handler_->GetDpiScale());
 }
 
+void FlutterWindowsView::RegisterKeyboardHookHandlers(
+    flutter::BinaryMessenger* messenger) {
+  AddKeyboardHookHandler(std::make_unique<flutter::KeyEventHandler>(messenger));
+  AddKeyboardHookHandler(
+      std::make_unique<flutter::TextInputPlugin>(messenger, this));
+}
+
+void FlutterWindowsView::AddKeyboardHookHandler(
+    std::unique_ptr<flutter::KeyboardHookHandler> handler) {
+  keyboard_hook_handlers_.push_back(std::move(handler));
+}
+
 uint32_t FlutterWindowsView::GetFrameBufferId(size_t width, size_t height) {
   // Called on an engine-controlled (non-platform) thread.
   std::unique_lock<std::mutex> lock(resize_mutex_);
@@ -120,11 +129,12 @@ void FlutterWindowsView::OnText(const std::u16string& text) {
   SendText(text);
 }
 
-void FlutterWindowsView::OnKey(int key,
+bool FlutterWindowsView::OnKey(int key,
                                int scancode,
                                int action,
-                               char32_t character) {
-  SendKey(key, scancode, action, character);
+                               char32_t character,
+                               bool extended) {
+  return SendKey(key, scancode, action, character, extended);
 }
 
 void FlutterWindowsView::OnScroll(double x,
@@ -215,13 +225,19 @@ void FlutterWindowsView::SendText(const std::u16string& text) {
   }
 }
 
-void FlutterWindowsView::SendKey(int key,
+bool FlutterWindowsView::SendKey(int key,
                                  int scancode,
                                  int action,
-                                 char32_t character) {
+                                 char32_t character,
+                                 bool extended) {
   for (const auto& handler : keyboard_hook_handlers_) {
-    handler->KeyboardHook(this, key, scancode, action, character);
+    if (handler->KeyboardHook(this, key, scancode, action, character,
+                              extended)) {
+      // key event was handled, so don't send to other handlers.
+      return true;
+    }
   }
+  return false;
 }
 
 void FlutterWindowsView::SendScroll(double x,

shell/platform/windows/flutter_windows_view.h

@@ -38,14 +38,14 @@ inline constexpr uint32_t kWindowFrameBufferID = 0;
 class FlutterWindowsView : public WindowBindingHandlerDelegate,
                            public TextInputPluginDelegate {
  public:
-  // Creates a FlutterWindowsView with the given implementator of
+  // Creates a FlutterWindowsView with the given implementor of
   // WindowBindingHandler.
   //
   // In order for object to render Flutter content the SetEngine method must be
   // called with a valid FlutterWindowsEngine instance.
   FlutterWindowsView(std::unique_ptr<WindowBindingHandler> window_binding);
 
-  ~FlutterWindowsView();
+  virtual ~FlutterWindowsView();
 
   // Configures the window instance with an instance of a running Flutter
   // engine.
@@ -100,7 +100,11 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate,
   void OnText(const std::u16string&) override;
 
   // |WindowBindingHandlerDelegate|
-  void OnKey(int key, int scancode, int action, char32_t character) override;
+  bool OnKey(int key,
+             int scancode,
+             int action,
+             char32_t character,
+             bool extended) override;
 
   // |WindowBindingHandlerDelegate|
   void OnScroll(double x,
@@ -112,6 +116,15 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate,
   // |TextInputPluginDelegate|
   void OnCursorRectUpdated(const Rect& rect) override;
 
+ protected:
+  // Called to create the keyboard hook handlers.
+  virtual void RegisterKeyboardHookHandlers(
+      flutter::BinaryMessenger* messenger);
+
+  // Used by RegisterKeyboardHookHandlers to add a new keyboard hook handler.
+  void AddKeyboardHookHandler(
+      std::unique_ptr<flutter::KeyboardHookHandler> handler);
+
  private:
   // Struct holding the mouse state. The engine doesn't keep track of which
   // mouse buttons have been pressed, so it's the embedding's responsibility.
@@ -166,7 +179,11 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate,
   void SendText(const std::u16string&);
 
   // Reports a raw keyboard message to Flutter engine.
-  void SendKey(int key, int scancode, int action, char32_t character);
+  bool SendKey(int key,
+               int scancode,
+               int action,
+               char32_t character,
+               bool extended);
 
   // Reports scroll wheel events to Flutter engine.
   void SendScroll(double x,

shell/platform/windows/key_event_handler.cc

@@ -10,6 +10,10 @@
 
 #include "flutter/shell/platform/common/cpp/json_message_codec.h"
 
+namespace flutter {
+
+namespace {
+
 static constexpr char kChannelName[] = "flutter/keyevent";
 
 static constexpr char kKeyCodeKey[] = "keyCode";
@@ -18,33 +22,35 @@ static constexpr char kCharacterCodePointKey[] = "characterCodePoint";
 static constexpr char kModifiersKey[] = "modifiers";
 static constexpr char kKeyMapKey[] = "keymap";
 static constexpr char kTypeKey[] = "type";
+static constexpr char kHandledKey[] = "handled";
 
 static constexpr char kWindowsKeyMap[] = "windows";
 static constexpr char kKeyUp[] = "keyup";
 static constexpr char kKeyDown[] = "keydown";
 
-namespace flutter {
+// The maximum number of pending events to keep before
+// emitting a warning on the console about unhandled events.
+static constexpr int kMaxPendingEvents = 1000;
 
 // Re-definition of the modifiers for compatibility with the Flutter framework.
 // These have to be in sync with the framework's RawKeyEventDataWindows
 // modifiers definition.
 // https://github.com/flutter/flutter/blob/19ff596979e407c484a32f4071420fca4f4c885f/packages/flutter/lib/src/services/raw_keyboard_windows.dart#L203
-const int kShift = 1 << 0;
-const int kShiftLeft = 1 << 1;
-const int kShiftRight = 1 << 2;
-const int kControl = 1 << 3;
-const int kControlLeft = 1 << 4;
-const int kControlRight = 1 << 5;
-const int kAlt = 1 << 6;
-const int kAltLeft = 1 << 7;
-const int kAltRight = 1 << 8;
-const int kWinLeft = 1 << 9;
-const int kWinRight = 1 << 10;
-const int kCapsLock = 1 << 11;
-const int kNumLock = 1 << 12;
-const int kScrollLock = 1 << 13;
+static constexpr int kShift = 1 << 0;
+static constexpr int kShiftLeft = 1 << 1;
+static constexpr int kShiftRight = 1 << 2;
+static constexpr int kControl = 1 << 3;
+static constexpr int kControlLeft = 1 << 4;
+static constexpr int kControlRight = 1 << 5;
+static constexpr int kAlt = 1 << 6;
+static constexpr int kAltLeft = 1 << 7;
+static constexpr int kAltRight = 1 << 8;
+static constexpr int kWinLeft = 1 << 9;
+static constexpr int kWinRight = 1 << 10;
+static constexpr int kCapsLock = 1 << 11;
+static constexpr int kNumLock = 1 << 12;
+static constexpr int kScrollLock = 1 << 13;
 
-namespace {
 /// Calls GetKeyState() an all modifier keys and packs the result in an int,
 /// with the re-defined values declared above for compatibility with the Flutter
 /// framework.
@@ -81,25 +87,130 @@ int GetModsForKeyState() {
     mods |= kScrollLock;
   return mods;
 }
+
+// This uses event data instead of generating a serial number because
+// information can't be attached to the redispatched events, so it has to be
+// possible to compute an ID from the identifying data in the event when it is
+// received again in order to differentiate between events that are new, and
+// events that have been redispatched.
+//
+// Another alternative would be to compute a checksum from all the data in the
+// event (just compute it over the bytes in the struct, probably skipping
+// timestamps), but the fields used below are enough to differentiate them, and
+// since Windows does some processing on the events (coming up with virtual key
+// codes, setting timestamps, etc.), it's not clear that the redispatched
+// events would have the same checksums.
+uint64_t CalculateEventId(int scancode, int action, bool extended) {
+  // Calculate a key event ID based on the scan code of the key pressed,
+  // and the flags we care about.
+  return scancode | (((action == WM_KEYUP ? KEYEVENTF_KEYUP : 0x0) |
+                      (extended ? KEYEVENTF_EXTENDEDKEY : 0x0))
+                     << 16);
+}
+
 }  // namespace
 
-KeyEventHandler::KeyEventHandler(flutter::BinaryMessenger* messenger)
+KeyEventHandler::KeyEventHandler(flutter::BinaryMessenger* messenger,
+                                 KeyEventHandler::SendInputDelegate send_input)
     : channel_(
           std::make_unique<flutter::BasicMessageChannel<rapidjson::Document>>(
               messenger,
               kChannelName,
-              &flutter::JsonMessageCodec::GetInstance())) {}
+              &flutter::JsonMessageCodec::GetInstance())),
+      send_input_(send_input) {
+  assert(send_input != nullptr);
+}
 
 KeyEventHandler::~KeyEventHandler() = default;
 
 void KeyEventHandler::TextHook(FlutterWindowsView* view,
                                const std::u16string& code_point) {}
 
-void KeyEventHandler::KeyboardHook(FlutterWindowsView* view,
+KEYBDINPUT* KeyEventHandler::FindPendingEvent(uint64_t id) {
+  if (pending_events_.empty()) {
+    return nullptr;
+  }
+  for (auto iter = pending_events_.begin(); iter != pending_events_.end();
+       ++iter) {
+    if (iter->first == id) {
+      return &iter->second;
+    }
+  }
+  return nullptr;
+}
+
+void KeyEventHandler::RemovePendingEvent(uint64_t id) {
+  for (auto iter = pending_events_.begin(); iter != pending_events_.end();
+       ++iter) {
+    if (iter->first == id) {
+      pending_events_.erase(iter);
+      return;
+    }
+  }
+  std::cerr << "Tried to remove pending event with id " << id
+            << ", but the event was not found." << std::endl;
+}
+
+void KeyEventHandler::AddPendingEvent(uint64_t id,
+                                      int scancode,
+                                      int action,
+                                      bool extended) {
+  if (pending_events_.size() > kMaxPendingEvents) {
+    std::cerr
+        << "There are " << pending_events_.size()
+        << " keyboard events that have not yet received a response from the "
+        << "framework. Are responses being sent?" << std::endl;
+  }
+  KEYBDINPUT key_event = KEYBDINPUT{0};
+  key_event.wScan = scancode;
+  key_event.dwFlags = KEYEVENTF_SCANCODE |
+                      (extended ? KEYEVENTF_EXTENDEDKEY : 0x0) |
+                      (action == WM_KEYUP ? KEYEVENTF_KEYUP : 0x0);
+  pending_events_.push_back(std::make_pair(id, key_event));
+}
+
+void KeyEventHandler::HandleResponse(bool handled,
+                                     uint64_t id,
+                                     int action,
+                                     bool extended,
+                                     int scancode,
+                                     int character) {
+  if (handled) {
+    this->RemovePendingEvent(id);
+  } else {
+    // Since the framework didn't handle the event, we inject a newly
+    // synthesized one. We let Windows figure out the virtual key and
+    // character for the given scancode, as well as a new timestamp.
+    const KEYBDINPUT* key_event = this->FindPendingEvent(id);
+    if (key_event == nullptr) {
+      std::cerr << "Unable to find event " << id << " in pending events queue.";
+      return;
+    }
+    INPUT input_event;
+    input_event.type = INPUT_KEYBOARD;
+    input_event.ki = *key_event;
+    UINT accepted = send_input_(1, &input_event, sizeof(input_event));
+    if (accepted != 1) {
+      std::cerr << "Unable to synthesize event for unhandled keyboard event "
+                   "with scancode "
+                << scancode << " (character " << character << ")" << std::endl;
+    }
+  }
+}
+
+bool KeyEventHandler::KeyboardHook(FlutterWindowsView* view,
                                    int key,
                                    int scancode,
                                    int action,
-                                   char32_t character) {
+                                   char32_t character,
+                                   bool extended) {
+  const uint64_t id = CalculateEventId(scancode, action, extended);
+  if (FindPendingEvent(id) != nullptr) {
+    // Don't pass messages that we synthesized to the framework again.
+    RemovePendingEvent(id);
+    return false;
+  }
+
   // TODO: Translate to a cross-platform key code system rather than passing
   // the native key code.
   rapidjson::Document event(rapidjson::kObjectType);
@@ -119,9 +230,17 @@ void KeyEventHandler::KeyboardHook(FlutterWindowsView* view,
       break;
     default:
       std::cerr << "Unknown key event action: " << action << std::endl;
-      return;
+      return false;
   }
-  channel_->Send(event);
+  AddPendingEvent(id, scancode, action, extended);
+  channel_->Send(event, [this, id, action, extended, scancode, character](
+                            const uint8_t* reply, size_t reply_size) {
+    auto decoded = flutter::JsonMessageCodec::GetInstance().DecodeMessage(
+        reply, reply_size);
+    bool handled = (*decoded)[kHandledKey].GetBool();
+    this->HandleResponse(handled, id, action, extended, scancode, character);
+  });
+  return true;
 }
 
 }  // namespace flutter