From 9d6070dd18f4cdedcbb01851b60d3677860ef13c Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Thu, 21 Oct 2021 11:42:45 -0700 Subject: [PATCH] Win32: Enable semantics on accessibility query (flutter/engine#29269) Windows does not provide a system notification the embedder can subscribe to to determine when a screen reader or other assistive technology that requires the semantics tree has been enabled. [1] In the absence of such a notification, we watch for queries for the IAccessible COM object representing the root node of the semantics tree, and on receipt of such queries we enable the semantics tree. For the time being, we assume that if accessiblity is enabled, it will be enabled for the duration of the application lifetime. In a future patch we can optimise this by adopting the approach taken by Chromium, which is to disable semantics if we haven't received a query within some reasonable timeout. This patch enables Flutter's semantics tree; a follow-up patch wires in the AccessibilityBridge which maintains the AXTree of COM objects that is the Windows representation of the Flutter SemanticsTree. Issue: https://github.com/flutter/flutter/issues/77838 [1]: https://docs.microsoft.com/en-us/windows/win32/winauto/screen-reader-parameter --- .../platform/windows/flutter_window_win32.cc | 4 +++ .../platform/windows/flutter_window_win32.h | 3 ++ .../windows/flutter_window_win32_unittests.cc | 1 + .../windows/flutter_windows_engine.cc | 7 +++++ .../platform/windows/flutter_windows_engine.h | 8 +++++ .../platform/windows/flutter_windows_view.cc | 4 +++ .../platform/windows/flutter_windows_view.h | 6 ++++ .../windows/flutter_windows_view_unittests.cc | 19 ++++++++++++ .../windows/testing/mock_window_win32.h | 1 + .../windows/window_binding_handler_delegate.h | 4 +++ .../shell/platform/windows/window_win32.cc | 30 +++++++++++++++++++ .../shell/platform/windows/window_win32.h | 11 +++++++ 12 files changed, 98 insertions(+) diff --git a/engine/src/flutter/shell/platform/windows/flutter_window_win32.cc b/engine/src/flutter/shell/platform/windows/flutter_window_win32.cc index 31d551a59a..07a71e8ab6 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_window_win32.cc +++ b/engine/src/flutter/shell/platform/windows/flutter_window_win32.cc @@ -211,6 +211,10 @@ void FlutterWindowWin32::OnComposeChange(const std::u16string& text, binding_handler_delegate_->OnComposeChange(text, cursor_pos); } +void FlutterWindowWin32::OnUpdateSemanticsEnabled(bool enabled) { + binding_handler_delegate_->OnUpdateSemanticsEnabled(enabled); +} + void FlutterWindowWin32::OnScroll(double delta_x, double delta_y, FlutterPointerDeviceKind device_kind, diff --git a/engine/src/flutter/shell/platform/windows/flutter_window_win32.h b/engine/src/flutter/shell/platform/windows/flutter_window_win32.h index f1f13f37d8..bb867e3b93 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_window_win32.h +++ b/engine/src/flutter/shell/platform/windows/flutter_window_win32.h @@ -92,6 +92,9 @@ class FlutterWindowWin32 : public WindowWin32, public WindowBindingHandler { // |FlutterWindowBindingHandler| void OnResetImeComposing() override; + // |WindowWin32| + void OnUpdateSemanticsEnabled(bool enabled) override; + // |WindowWin32| void OnScroll(double delta_x, double delta_y, diff --git a/engine/src/flutter/shell/platform/windows/flutter_window_win32_unittests.cc b/engine/src/flutter/shell/platform/windows/flutter_window_win32_unittests.cc index a5e6683ce5..bdca8999e1 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_window_win32_unittests.cc +++ b/engine/src/flutter/shell/platform/windows/flutter_window_win32_unittests.cc @@ -216,6 +216,7 @@ class MockWindowBindingHandlerDelegate : public WindowBindingHandlerDelegate { MOCK_METHOD0(OnComposeCommit, void()); MOCK_METHOD0(OnComposeEnd, void()); MOCK_METHOD2(OnComposeChange, void(const std::u16string&, int)); + MOCK_METHOD1(OnUpdateSemanticsEnabled, void(bool)); MOCK_METHOD7(OnScroll, void(double, double, diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc index 7cf66667f0..b6fb9f4b7b 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc +++ b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.cc @@ -452,4 +452,11 @@ bool FlutterWindowsEngine::DispatchSemanticsAction( engine_, target, action, data.data(), data.size()) == kSuccess); } +void FlutterWindowsEngine::UpdateSemanticsEnabled(bool enabled) { + if (engine_ && semantics_enabled_ != enabled) { + semantics_enabled_ = enabled; + embedder_api_.UpdateSemanticsEnabled(engine_, enabled); + } +} + } // namespace flutter diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.h b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.h index 15decb5369..22e75149e9 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows_engine.h +++ b/engine/src/flutter/shell/platform/windows/flutter_windows_engine.h @@ -149,6 +149,12 @@ class FlutterWindowsEngine { FlutterSemanticsAction action, const std::vector& data); + // Informs the engine that the semantics enabled state has changed. + void UpdateSemanticsEnabled(bool enabled); + + // Returns true if the semantics tree is enabled. + bool semantics_enabled() const { return semantics_enabled_; } + private: // Allows swapping out embedder_api_ calls in tests. friend class EngineModifier; @@ -206,6 +212,8 @@ class FlutterWindowsEngine { FlutterDesktopOnPluginRegistrarDestroyed plugin_registrar_destruction_callback_ = nullptr; + bool semantics_enabled_ = false; + #ifndef WINUWP // The manager for WindowProc delegate registration and callbacks. std::unique_ptr window_proc_delegate_manager_; diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows_view.cc b/engine/src/flutter/shell/platform/windows/flutter_windows_view.cc index 35cb908beb..91783fcdfa 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows_view.cc +++ b/engine/src/flutter/shell/platform/windows/flutter_windows_view.cc @@ -248,6 +248,10 @@ void FlutterWindowsView::OnPlatformBrightnessChanged() { SendPlatformBrightnessChanged(); } +void FlutterWindowsView::OnUpdateSemanticsEnabled(bool enabled) { + engine_->UpdateSemanticsEnabled(enabled); +} + void FlutterWindowsView::OnCursorRectUpdated(const Rect& rect) { binding_handler_->OnCursorRectUpdated(rect); } diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows_view.h b/engine/src/flutter/shell/platform/windows/flutter_windows_view.h index 1dd820d815..84875d603a 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows_view.h +++ b/engine/src/flutter/shell/platform/windows/flutter_windows_view.h @@ -158,6 +158,9 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate, // |WindowBindingHandlerDelegate| void OnPlatformBrightnessChanged() override; + // |WindowBindingHandlerDelegate| + virtual void OnUpdateSemanticsEnabled(bool enabled) override; + // |TextInputPluginDelegate| void OnCursorRectUpdated(const Rect& rect) override; @@ -346,6 +349,9 @@ class FlutterWindowsView : public WindowBindingHandlerDelegate, // Target for the window width. Valid when resize_pending_ is set. Guarded by // resize_mutex_. size_t resize_target_height_ = 0; + + // True when flutter's semantics tree is enabled. + bool semantics_enabled_ = false; }; } // namespace flutter diff --git a/engine/src/flutter/shell/platform/windows/flutter_windows_view_unittests.cc b/engine/src/flutter/shell/platform/windows/flutter_windows_view_unittests.cc index 72a4671638..77343522b2 100644 --- a/engine/src/flutter/shell/platform/windows/flutter_windows_view_unittests.cc +++ b/engine/src/flutter/shell/platform/windows/flutter_windows_view_unittests.cc @@ -35,6 +35,7 @@ struct TestResponseHandle { }; static bool test_response = false; +static bool semantics_enabled = false; constexpr uint64_t kKeyEventFromChannel = 0x11; constexpr uint64_t kKeyEventFromEmbedder = 0x22; @@ -126,5 +127,23 @@ TEST(FlutterWindowsViewTest, RestartClearsKeyboardState) { key_event_logs.clear(); } +TEST(FlutterWindowsViewTest, EnableSemantics) { + std::unique_ptr engine = GetTestEngine(); + EngineModifier modifier(engine.get()); + modifier.embedder_api().UpdateSemanticsEnabled = + [](FLUTTER_API_SYMBOL(FlutterEngine) engine, bool enabled) { + semantics_enabled = enabled; + return kSuccess; + }; + + auto window_binding_handler = + std::make_unique<::testing::NiceMock>(); + FlutterWindowsView view(std::move(window_binding_handler)); + view.SetEngine(std::move(engine)); + + view.OnUpdateSemanticsEnabled(true); + EXPECT_TRUE(semantics_enabled); +} + } // namespace testing } // namespace flutter diff --git a/engine/src/flutter/shell/platform/windows/testing/mock_window_win32.h b/engine/src/flutter/shell/platform/windows/testing/mock_window_win32.h index b53e9f7c69..21d6840704 100644 --- a/engine/src/flutter/shell/platform/windows/testing/mock_window_win32.h +++ b/engine/src/flutter/shell/platform/windows/testing/mock_window_win32.h @@ -44,6 +44,7 @@ class MockWin32Window : public WindowWin32, public MockMessageQueue { MOCK_METHOD0(OnSetCursor, void()); MOCK_METHOD1(OnText, void(const std::u16string&)); MOCK_METHOD6(OnKey, bool(int, int, int, char32_t, bool, bool)); + MOCK_METHOD1(OnUpdateSemanticsEnabled, void(bool)); MOCK_METHOD4(OnScroll, void(double, double, FlutterPointerDeviceKind, int32_t)); MOCK_METHOD0(OnComposeBegin, void()); diff --git a/engine/src/flutter/shell/platform/windows/window_binding_handler_delegate.h b/engine/src/flutter/shell/platform/windows/window_binding_handler_delegate.h index 7bdedc9d4d..3b58e4aed9 100644 --- a/engine/src/flutter/shell/platform/windows/window_binding_handler_delegate.h +++ b/engine/src/flutter/shell/platform/windows/window_binding_handler_delegate.h @@ -98,6 +98,10 @@ class WindowBindingHandlerDelegate { // Notifies delegate that backing window has received brightness change event. virtual void OnPlatformBrightnessChanged() = 0; + + // Notifies delegate that the Flutter semantics tree should be enabled or + // disabled. + virtual void OnUpdateSemanticsEnabled(bool enabled) = 0; }; } // namespace flutter diff --git a/engine/src/flutter/shell/platform/windows/window_win32.cc b/engine/src/flutter/shell/platform/windows/window_win32.cc index 7735669e05..5dcb2a618d 100644 --- a/engine/src/flutter/shell/platform/windows/window_win32.cc +++ b/engine/src/flutter/shell/platform/windows/window_win32.cc @@ -139,6 +139,33 @@ void WindowWin32::TrackMouseLeaveEvent(HWND hwnd) { } } +void WindowWin32::OnGetObject(UINT const message, + WPARAM const wparam, + LPARAM const lparam) { + LRESULT reference_result = static_cast(0L); + + // Only the lower 32 bits of lparam are valid when checking the object id + // because it sometimes gets sign-extended incorrectly (but not always). + DWORD obj_id = static_cast(static_cast(lparam)); + + bool is_msaa_request = static_cast(OBJID_CLIENT) == obj_id; + if (is_msaa_request) { + // On Windows, we don't get a notification that the screen reader has been + // enabled or disabled. There is an API to query for screen reader state, + // but that state isn't set by all screen readers, including by Narrator, + // the screen reader that ships with Windows: + // https://docs.microsoft.com/en-us/windows/win32/winauto/screen-reader-parameter + // + // Instead, we enable semantics in Flutter if Windows issues queries for + // Microsoft Active Accessibility (MSAA) COM objects. + OnUpdateSemanticsEnabled(true); + + // TODO(cbracken): https://github.com/flutter/flutter/issues/77838 + // Once AccessibilityBridge is wired up, look up the IAccessible + // representing the root view and call LresultFromObject. + } +} + void WindowWin32::OnImeSetContext(UINT const message, WPARAM const wparam, LPARAM const lparam) { @@ -377,6 +404,9 @@ WindowWin32::HandleMessage(UINT const message, static_cast(WHEEL_DELTA)), 0.0, kFlutterPointerDeviceKindMouse, kDefaultPointerDeviceId); break; + case WM_GETOBJECT: + OnGetObject(message, wparam, lparam); + break; case WM_INPUTLANGCHANGE: // TODO(cbracken): pass this to TextInputManager to aid with // language-specific issues. diff --git a/engine/src/flutter/shell/platform/windows/window_win32.h b/engine/src/flutter/shell/platform/windows/window_win32.h index 3607faf820..d2ac6194f1 100644 --- a/engine/src/flutter/shell/platform/windows/window_win32.h +++ b/engine/src/flutter/shell/platform/windows/window_win32.h @@ -121,6 +121,14 @@ class WindowWin32 { bool extended, bool was_down) = 0; + // Called when the OS requests a COM object. + // + // The primary use of this function is to supply Windows with wrapped + // semantics objects for use by Windows accessibility. + void OnGetObject(UINT const message, + WPARAM const wparam, + LPARAM const lparam); + // Called when IME composing begins. virtual void OnComposeBegin() = 0; @@ -170,6 +178,9 @@ class WindowWin32 { // |rect| is in Win32 window coordinates. virtual void UpdateCursorRect(const Rect& rect); + // Called when accessibility support is enabled or disabled. + virtual void OnUpdateSemanticsEnabled(bool enabled) = 0; + // Called when mouse scrollwheel input occurs. virtual void OnScroll(double delta_x, double delta_y,