From 258cde6cd2381973562702960e9513e85e708bcb Mon Sep 17 00:00:00 2001 From: Bruno Leroux Date: Fri, 20 Jan 2023 11:43:16 +0100 Subject: [PATCH] [macos] Synthesize modifier keys events on pointer events (flutter/engine#37870) * [macos] Synthesize modifier keys events on pointer events * Move test to FlutterViewControllerTest * Simplify by using 'for in' Co-authored-by: Bruno Leroux --- .../Source/FlutterEmbedderKeyResponder.h | 9 ++ .../Source/FlutterEmbedderKeyResponder.mm | 13 +++ .../framework/Source/FlutterKeyboardManager.h | 9 ++ .../Source/FlutterKeyboardManager.mm | 8 ++ .../framework/Source/FlutterViewController.mm | 2 + .../Source/FlutterViewControllerTest.mm | 107 ++++++++++++++++++ 6 files changed, 148 insertions(+) diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.h b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.h index ab2b1c107d..7a3b9ee51b 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.h +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.h @@ -30,4 +30,13 @@ typedef void (^FlutterSendEmbedderKeyEvent)(const FlutterKeyEvent& /* event */, */ - (nonnull instancetype)initWithSendEvent:(_Nonnull FlutterSendEmbedderKeyEvent)sendEvent; +/** + * Synthesize modifier keys events. + * + * If needed, synthesize modifier keys up and down events by comparing their + * current pressing states with the given modifier flags. + */ +- (void)syncModifiersIfNeeded:(NSEventModifierFlags)modifierFlags + timestamp:(NSTimeInterval)timestamp; + @end diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm index 5854bb02f3..4a5d14f4d8 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderKeyResponder.mm @@ -780,6 +780,19 @@ struct FlutterKeyPendingResponse { [_pendingResponses removeObjectForKey:@(responseId)]; } +- (void)syncModifiersIfNeeded:(NSEventModifierFlags)modifierFlags + timestamp:(NSTimeInterval)timestamp { + FlutterAsyncKeyCallback replyCallback = ^(BOOL handled) { + // Do nothing. + }; + FlutterKeyCallbackGuard* guardedCallback = + [[FlutterKeyCallbackGuard alloc] initWithCallback:replyCallback]; + [self synchronizeModifiers:modifierFlags + ignoringFlags:0 + timestamp:timestamp + guard:guardedCallback]; +} + @end namespace { diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.h b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.h index c8ba5058aa..64ca1a35f6 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.h +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.h @@ -48,4 +48,13 @@ */ - (BOOL)isDispatchingKeyEvent:(nonnull NSEvent*)event; +/** + * Synthesize modifier keys events. + * + * If needed, synthesize modifier keys up and down events by comparing their + * current pressing states with the given modifier flags. + */ +- (void)syncModifiersIfNeeded:(NSEventModifierFlags)modifierFlags + timestamp:(NSTimeInterval)timestamp; + @end diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm index 116c94537d..78c18dd0b0 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterKeyboardManager.mm @@ -320,4 +320,12 @@ typedef _Nullable _NSResponderPtr (^NextResponderProvider)(); } } +- (void)syncModifiersIfNeeded:(NSEventModifierFlags)modifierFlags + timestamp:(NSTimeInterval)timestamp { + // The embedder responder is the first element in _primaryResponders. + FlutterEmbedderKeyResponder* embedderResponder = + (FlutterEmbedderKeyResponder*)_primaryResponders[0]; + [embedderResponder syncModifiersIfNeeded:modifierFlags timestamp:timestamp]; +} + @end diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm index 9034e8deaa..2c9e76e659 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController.mm @@ -661,6 +661,8 @@ static void CommonInit(FlutterViewController* controller) { flutterEvent.scroll_delta_y = scaledDeltaY; } } + + [_keyboardManager syncModifiersIfNeeded:event.modifierFlags timestamp:event.timestamp]; [_engine sendPointerEvent:flutterEvent]; // Update tracking of state as reported to Flutter. diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm index 903065731a..dbc98870a0 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTest.mm @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#import "KeyCodeMap_Internal.h" #import "flutter/shell/platform/darwin/macos/framework/Headers/FlutterViewController.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewController_Internal.h" @@ -13,8 +14,27 @@ #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterRenderer.h" #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterViewControllerTestUtils.h" +#include "flutter/shell/platform/embedder/test_utils/key_codes.g.h" #import "flutter/testing/testing.h" +// A wrap to convert FlutterKeyEvent to a ObjC class. +@interface KeyEventWrapper : NSObject +@property(nonatomic) FlutterKeyEvent* data; +- (nonnull instancetype)initWithEvent:(const FlutterKeyEvent*)event; +@end + +@implementation KeyEventWrapper +- (instancetype)initWithEvent:(const FlutterKeyEvent*)event { + self = [super init]; + _data = new FlutterKeyEvent(*event); + return self; +} + +- (void)dealloc { + delete _data; +} +@end + @interface FlutterViewControllerTestObjC : NSObject - (bool)testKeyEventsAreSentToFramework; - (bool)testKeyEventsArePropagatedIfNotHandled; @@ -22,6 +42,7 @@ - (bool)testFlagsChangedEventsArePropagatedIfNotHandled; - (bool)testKeyboardIsRestartedOnEngineRestart; - (bool)testTrackpadGesturesAreSentToFramework; +- (bool)testModifierKeysAreSynthesizedOnMouseMove; - (bool)testViewWillAppearCalledMultipleTimes; - (bool)testFlutterViewIsConfigured; @@ -30,6 +51,8 @@ userData:(nullable void*)userData; @end +using namespace ::flutter::testing::keycodes; + namespace flutter::testing { namespace { @@ -69,6 +92,19 @@ NSResponder* mockResponder() { OCMStub([mock flagsChanged:[OCMArg any]]).andDo(nil); return mock; } + +NSEvent* CreateMouseEvent(NSEventModifierFlags modifierFlags) { + return [NSEvent mouseEventWithType:NSEventTypeMouseMoved + location:NSZeroPoint + modifierFlags:modifierFlags + timestamp:0 + windowNumber:0 + context:nil + eventNumber:0 + clickCount:1 + pressure:1.0]; +} + } // namespace TEST(FlutterViewController, HasViewThatHidesOtherViewsInAccessibility) { @@ -161,6 +197,10 @@ TEST(FlutterViewControllerTest, TestTrackpadGesturesAreSentToFramework) { ASSERT_TRUE([[FlutterViewControllerTestObjC alloc] testTrackpadGesturesAreSentToFramework]); } +TEST(FlutterViewControllerTest, TestModifierKeysAreSynthesizedOnMouseMove) { + ASSERT_TRUE([[FlutterViewControllerTestObjC alloc] testModifierKeysAreSynthesizedOnMouseMove]); +} + TEST(FlutterViewControllerTest, testViewWillAppearCalledMultipleTimes) { ASSERT_TRUE([[FlutterViewControllerTestObjC alloc] testViewWillAppearCalledMultipleTimes]); } @@ -763,4 +803,71 @@ TEST(FlutterViewControllerTest, testFlutterViewIsConfigured) { return true; } +- (bool)testModifierKeysAreSynthesizedOnMouseMove { + id engineMock = OCMClassMock([FlutterEngine class]); + // Need to return a real renderer to allow view controller to load. + FlutterRenderer* renderer_ = [[FlutterRenderer alloc] initWithFlutterEngine:engineMock]; + OCMStub([engineMock renderer]).andReturn(renderer_); + + // Capture calls to sendKeyEvent + __block NSMutableArray* events = + [[NSMutableArray alloc] init]; + OCMStub([[engineMock ignoringNonObjectArgs] sendKeyEvent:FlutterKeyEvent {} + callback:nil + userData:nil]) + .andDo((^(NSInvocation* invocation) { + FlutterKeyEvent* event; + [invocation getArgument:&event atIndex:2]; + [events addObject:[[KeyEventWrapper alloc] initWithEvent:event]]; + })); + + FlutterViewController* viewController = [[FlutterViewController alloc] initWithEngine:engineMock + nibName:@"" + bundle:nil]; + [viewController loadView]; + [engineMock setViewController:viewController]; + [viewController viewWillAppear]; + + // Zeroed modifier flag should not synthesize events. + NSEvent* mouseEvent = flutter::testing::CreateMouseEvent(0x00); + [viewController mouseMoved:mouseEvent]; + EXPECT_EQ([events count], 0u); + + // For each modifier key, check that key events are synthesized. + for (NSNumber* keyCode in flutter::keyCodeToModifierFlag) { + FlutterKeyEvent* event; + NSNumber* logicalKey; + NSNumber* physicalKey; + NSNumber* flag = flutter::keyCodeToModifierFlag[keyCode]; + + // Should synthesize down event. + NSEvent* mouseEvent = flutter::testing::CreateMouseEvent([flag unsignedLongValue]); + [viewController mouseMoved:mouseEvent]; + EXPECT_EQ([events count], 1u); + event = events[0].data; + logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; + physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; + EXPECT_EQ(event->type, kFlutterKeyEventTypeDown); + EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); + EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); + EXPECT_EQ(event->synthesized, true); + + // Should synthesize up event. + mouseEvent = flutter::testing::CreateMouseEvent(0x00); + [viewController mouseMoved:mouseEvent]; + EXPECT_EQ([events count], 2u); + event = events[1].data; + logicalKey = [flutter::keyCodeToLogicalKey objectForKey:keyCode]; + physicalKey = [flutter::keyCodeToPhysicalKey objectForKey:keyCode]; + EXPECT_EQ(event->type, kFlutterKeyEventTypeUp); + EXPECT_EQ(event->logical, logicalKey.unsignedLongLongValue); + EXPECT_EQ(event->physical, physicalKey.unsignedLongLongValue); + EXPECT_EQ(event->synthesized, true); + + [events removeAllObjects]; + }; + + return true; +} + @end