Win32: Enable semantics on accessibility query (flutter/engine#29269)

Windows does not provide a system notification the embedder can
subscribe to to determine when a screen reader or other assistive
technology that requires the semantics tree has been enabled. [1]

In the absence of such a notification, we watch for queries for the
IAccessible COM object representing the root node of the semantics tree,
and on receipt of such queries we enable the semantics tree. For the
time being, we assume that if accessiblity is enabled, it will be
enabled for the duration of the application lifetime. In a future patch
we can optimise this by adopting the approach taken by Chromium, which
is to disable semantics if we haven't received a query within some
reasonable timeout.

This patch enables Flutter's semantics tree; a follow-up patch wires in
the AccessibilityBridge which maintains the AXTree of COM objects that
is the Windows representation of the Flutter SemanticsTree.

Issue: https://github.com/flutter/flutter/issues/77838

[1]: https://docs.microsoft.com/en-us/windows/win32/winauto/screen-reader-parameter

engine/src/flutter/shell/platform/windows/flutter_window_win32.cc

@@ -211,6 +211,10 @@ void FlutterWindowWin32::OnComposeChange(const std::u16string& text,
   binding_handler_delegate_->OnComposeChange(text, cursor_pos);
 }
 
+void FlutterWindowWin32::OnUpdateSemanticsEnabled(bool enabled) {
+  binding_handler_delegate_->OnUpdateSemanticsEnabled(enabled);
+}
+
 void FlutterWindowWin32::OnScroll(double delta_x,
                                   double delta_y,
                                   FlutterPointerDeviceKind device_kind,

engine/src/flutter/shell/platform/windows/flutter_window_win32.h

@@ -92,6 +92,9 @@ class FlutterWindowWin32 : public WindowWin32, public WindowBindingHandler {
   // |FlutterWindowBindingHandler|
   void OnResetImeComposing() override;
 
+  // |WindowWin32|
+  void OnUpdateSemanticsEnabled(bool enabled) override;
+
   // |WindowWin32|
   void OnScroll(double delta_x,
                 double delta_y,

engine/src/flutter/shell/platform/windows/flutter_window_win32_unittests.cc

@@ -216,6 +216,7 @@ class MockWindowBindingHandlerDelegate : public WindowBindingHandlerDelegate {
   MOCK_METHOD0(OnComposeCommit, void());
   MOCK_METHOD0(OnComposeEnd, void());
   MOCK_METHOD2(OnComposeChange, void(const std::u16string&, int));
+  MOCK_METHOD1(OnUpdateSemanticsEnabled, void(bool));
   MOCK_METHOD7(OnScroll,
                void(double,
                     double,

engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc

@@ -452,4 +452,11 @@ bool FlutterWindowsEngine::DispatchSemanticsAction(
               engine_, target, action, data.data(), data.size()) == kSuccess);
 }
 
+void FlutterWindowsEngine::UpdateSemanticsEnabled(bool enabled) {
+  if (engine_ && semantics_enabled_ != enabled) {
+    semantics_enabled_ = enabled;
+    embedder_api_.UpdateSemanticsEnabled(engine_, enabled);
+  }
+}
+
 }  // namespace flutter

engine/src/flutter/shell/platform/windows/flutter_windows_engine.h

@@ -149,6 +149,12 @@ class FlutterWindowsEngine {
                                FlutterSemanticsAction action,
                                const std::vector<uint8_t>& data);
 
+  // Informs the engine that the semantics enabled state has changed.
+  void UpdateSemanticsEnabled(bool enabled);
+
+  // Returns true if the semantics tree is enabled.
+  bool semantics_enabled() const { return semantics_enabled_; }
+
  private:
   // Allows swapping out embedder_api_ calls in tests.
   friend class EngineModifier;
@@ -206,6 +212,8 @@ class FlutterWindowsEngine {
   FlutterDesktopOnPluginRegistrarDestroyed
       plugin_registrar_destruction_callback_ = nullptr;
 
+  bool semantics_enabled_ = false;
+
 #ifndef WINUWP
   // The manager for WindowProc delegate registration and callbacks.
   std::unique_ptr<WindowProcDelegateManagerWin32> window_proc_delegate_manager_;

engine/src/flutter/shell/platform/windows/flutter_windows_view.cc

@@ -248,6 +248,10 @@ void FlutterWindowsView::OnPlatformBrightnessChanged() {
   SendPlatformBrightnessChanged();
 }
 
+void FlutterWindowsView::OnUpdateSemanticsEnabled(bool enabled) {
+  engine_->UpdateSemanticsEnabled(enabled);
+}
+
 void FlutterWindowsView::OnCursorRectUpdated(const Rect& rect) {
   binding_handler_->OnCursorRectUpdated(rect);
 }

engine/src/flutter/shell/platform/windows/flutter_windows_view.h

@@ -158,6 +158,9 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate,
   // |WindowBindingHandlerDelegate|
   void OnPlatformBrightnessChanged() override;
 
+  // |WindowBindingHandlerDelegate|
+  virtual void OnUpdateSemanticsEnabled(bool enabled) override;
+
   // |TextInputPluginDelegate|
   void OnCursorRectUpdated(const Rect& rect) override;
 
@@ -346,6 +349,9 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate,
   // Target for the window width. Valid when resize_pending_ is set. Guarded by
   // resize_mutex_.
   size_t resize_target_height_ = 0;
+
+  // True when flutter's semantics tree is enabled.
+  bool semantics_enabled_ = false;
 };
 
 }  // namespace flutter

engine/src/flutter/shell/platform/windows/flutter_windows_view_unittests.cc

@@ -35,6 +35,7 @@ struct TestResponseHandle {
 };
 
 static bool test_response = false;
+static bool semantics_enabled = false;
 
 constexpr uint64_t kKeyEventFromChannel = 0x11;
 constexpr uint64_t kKeyEventFromEmbedder = 0x22;
@@ -126,5 +127,23 @@ TEST(FlutterWindowsViewTest, RestartClearsKeyboardState) {
   key_event_logs.clear();
 }
 
+TEST(FlutterWindowsViewTest, EnableSemantics) {
+  std::unique_ptr<FlutterWindowsEngine> engine = GetTestEngine();
+  EngineModifier modifier(engine.get());
+  modifier.embedder_api().UpdateSemanticsEnabled =
+      [](FLUTTER_API_SYMBOL(FlutterEngine) engine, bool enabled) {
+        semantics_enabled = enabled;
+        return kSuccess;
+      };
+
+  auto window_binding_handler =
+      std::make_unique<::testing::NiceMock<MockWindowBindingHandler>>();
+  FlutterWindowsView view(std::move(window_binding_handler));
+  view.SetEngine(std::move(engine));
+
+  view.OnUpdateSemanticsEnabled(true);
+  EXPECT_TRUE(semantics_enabled);
+}
+
 }  // namespace testing
 }  // namespace flutter

engine/src/flutter/shell/platform/windows/testing/mock_window_win32.h

@@ -44,6 +44,7 @@ class MockWin32Window : public WindowWin32, public MockMessageQueue {
   MOCK_METHOD0(OnSetCursor, void());
   MOCK_METHOD1(OnText, void(const std::u16string&));
   MOCK_METHOD6(OnKey, bool(int, int, int, char32_t, bool, bool));
+  MOCK_METHOD1(OnUpdateSemanticsEnabled, void(bool));
   MOCK_METHOD4(OnScroll,
                void(double, double, FlutterPointerDeviceKind, int32_t));
   MOCK_METHOD0(OnComposeBegin, void());

engine/src/flutter/shell/platform/windows/window_binding_handler_delegate.h

@@ -98,6 +98,10 @@ class WindowBindingHandlerDelegate {
 
   // Notifies delegate that backing window has received brightness change event.
   virtual void OnPlatformBrightnessChanged() = 0;
+
+  // Notifies delegate that the Flutter semantics tree should be enabled or
+  // disabled.
+  virtual void OnUpdateSemanticsEnabled(bool enabled) = 0;
 };
 
 }  // namespace flutter

engine/src/flutter/shell/platform/windows/window_win32.cc

@@ -139,6 +139,33 @@ void WindowWin32::TrackMouseLeaveEvent(HWND hwnd) {
   }
 }
 
+void WindowWin32::OnGetObject(UINT const message,
+                              WPARAM const wparam,
+                              LPARAM const lparam) {
+  LRESULT reference_result = static_cast<LRESULT>(0L);
+
+  // Only the lower 32 bits of lparam are valid when checking the object id
+  // because it sometimes gets sign-extended incorrectly (but not always).
+  DWORD obj_id = static_cast<DWORD>(static_cast<DWORD_PTR>(lparam));
+
+  bool is_msaa_request = static_cast<DWORD>(OBJID_CLIENT) == obj_id;
+  if (is_msaa_request) {
+    // On Windows, we don't get a notification that the screen reader has been
+    // enabled or disabled. There is an API to query for screen reader state,
+    // but that state isn't set by all screen readers, including by Narrator,
+    // the screen reader that ships with Windows:
+    // https://docs.microsoft.com/en-us/windows/win32/winauto/screen-reader-parameter
+    //
+    // Instead, we enable semantics in Flutter if Windows issues queries for
+    // Microsoft Active Accessibility (MSAA) COM objects.
+    OnUpdateSemanticsEnabled(true);
+
+    // TODO(cbracken): https://github.com/flutter/flutter/issues/77838
+    // Once AccessibilityBridge is wired up, look up the IAccessible
+    // representing the root view and call LresultFromObject.
+  }
+}
+
 void WindowWin32::OnImeSetContext(UINT const message,
                                   WPARAM const wparam,
                                   LPARAM const lparam) {
@@ -377,6 +404,9 @@ WindowWin32::HandleMessage(UINT const message,
                 static_cast<double>(WHEEL_DELTA)),
                0.0, kFlutterPointerDeviceKindMouse, kDefaultPointerDeviceId);
       break;
+    case WM_GETOBJECT:
+      OnGetObject(message, wparam, lparam);
+      break;
     case WM_INPUTLANGCHANGE:
       // TODO(cbracken): pass this to TextInputManager to aid with
       // language-specific issues.

engine/src/flutter/shell/platform/windows/window_win32.h

@@ -121,6 +121,14 @@ class WindowWin32 {
                      bool extended,
                      bool was_down) = 0;
 
+  // Called when the OS requests a COM object.
+  //
+  // The primary use of this function is to supply Windows with wrapped
+  // semantics objects for use by Windows accessibility.
+  void OnGetObject(UINT const message,
+                   WPARAM const wparam,
+                   LPARAM const lparam);
+
   // Called when IME composing begins.
   virtual void OnComposeBegin() = 0;
 
@@ -170,6 +178,9 @@ class WindowWin32 {
   // |rect| is in Win32 window coordinates.
   virtual void UpdateCursorRect(const Rect& rect);
 
+  // Called when accessibility support is enabled or disabled.
+  virtual void OnUpdateSemanticsEnabled(bool enabled) = 0;
+
   // Called when mouse scrollwheel input occurs.
   virtual void OnScroll(double delta_x,
                         double delta_y,