[ios][ios17][text_input]fix text input system highlight in iOS 17 Beta 7 with firstRectForRange (flutter/engine#45303)

Design doc with more details: https://docs.google.com/document/d/1sM3HMv-SQin39yX1aPUU7vtGv7Hcef1Quc3QhRXBl6A/edit?resourcekey=0-SFYD8vmOIkXiXCZvB1Wlcw#heading=h.tul5o3hopauh Just to be safe, for now we only change for iOS 17. However, the same logic should apply to older iOS as well. So will update in a separate PR. Basically we were using `firstRectForRange` incorrectly. According to [the API doc](https://developer.apple.com/documentation/uikit/uitextinput/1614570-firstrectforrange), `firstRectForRange` should return **the rect for the queried range, or a subrange through the end of line, if the range encompasses multiple lines**. For LTR language: ``` 0, 1, 2, 3 4, 5, 6, 7 ``` Query: `[1, 2, 3, 4]` Returns the following rect denoted by `[ ]`: ``` 0, [1, 2, 3] 4, 5, 6, 7 ``` Similarly, for RTL language (Arabic): ``` 3, 2, 1, 0 7, 6, 5, 4 ``` Query: `[1, 2, 3, 4]` Returns the following rect denoted by `[ ]`: ``` [3, 2, 1], 0 7, 6, 5, 4 ``` *List which issues are fixed by this PR. You must list at least one issue.* https://github.com/flutter/flutter/issues/131622 *If you had to change anything in the [flutter/tests] repo, include a link to the migration guide as per the [breaking change policy].* [C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style
2023-09-01 12:03:48 -07:00
parent 78f1c64cb8
commit ef55ffaa9b
2 changed files with 286 additions and 9 deletions
--- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm
+++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm
@@ -1617,6 +1617,8 @@ static BOOL IsSelectionRectBoundaryCloserToPoint(CGPoint point,
 // and to position the
 // candidates view for multi-stage input methods (e.g., Japanese) when using a
 // physical keyboard.
+// Returns the rect for the queried range, or a subrange through the end of line, if
+// the range encompasses multiple lines.
 - (CGRect)firstRectForRange:(UITextRange*)range {
  NSAssert([range.start isKindOfClass:[FlutterTextPosition class]],
           @"Expected a FlutterTextPosition for range.start (got %@).", [range.start class]);
@@ -1671,6 +1673,14 @@ static BOOL IsSelectionRectBoundaryCloserToPoint(CGPoint point,
  if (end < start) {
    first = end;
  }
+
+  CGRect startSelectionRect = CGRectNull;
+  CGRect endSelectionRect = CGRectNull;
+  // Selection rects from different langauges may have different minY/maxY.
+  // So we need to iterate through each rects to update minY/maxY.
+  CGFloat minY = CGFLOAT_MAX;
+  CGFloat maxY = CGFLOAT_MIN;
+
  FlutterTextRange* textRange = [FlutterTextRange
      rangeWithNSRange:fml::RangeForCharactersInRange(self.text, NSMakeRange(0, self.text.length))];
  for (NSUInteger i = 0; i < [_selectionRects count]; i++) {
@@ -1681,11 +1691,38 @@ static BOOL IsSelectionRectBoundaryCloserToPoint(CGPoint point,
        !isLastSelectionRect && _selectionRects[i + 1].position > first;
    if (startsOnOrBeforeStartOfRange &&
        (endOfTextIsAfterStartOfRange || nextSelectionRectIsAfterStartOfRange)) {
-      return _selectionRects[i].rect;
+      // TODO(hellohaunlin): Remove iOS 17 check. The logic should also work for older versions.
+      if (@available(iOS 17, *)) {
+        startSelectionRect = _selectionRects[i].rect;
+      } else {
+        return _selectionRects[i].rect;
+      }
+    }
+    if (!CGRectIsNull(startSelectionRect)) {
+      minY = fmin(minY, CGRectGetMinY(_selectionRects[i].rect));
+      maxY = fmax(maxY, CGRectGetMaxY(_selectionRects[i].rect));
+      BOOL endsOnOrAfterEndOfRange = _selectionRects[i].position >= end - 1;  // end is exclusive
+      BOOL nextSelectionRectIsOnNextLine =
+          !isLastSelectionRect &&
+          // Selection rects from different langauges in 2 lines may overlap with each other.
+          // A good approximation is to check if the center of next rect is below the bottom of
+          // current rect.
+          // TODO(hellohuanlin): Consider passing the line break info from framework.
+          CGRectGetMidY(_selectionRects[i + 1].rect) > CGRectGetMaxY(_selectionRects[i].rect);
+      if (endsOnOrAfterEndOfRange || isLastSelectionRect || nextSelectionRectIsOnNextLine) {
+        endSelectionRect = _selectionRects[i].rect;
+        break;
+      }
    }
  }
-
-  return CGRectZero;
+  if (CGRectIsNull(startSelectionRect) || CGRectIsNull(endSelectionRect)) {
+    return CGRectZero;
+  } else {
+    // fmin/fmax to support both LTR and RTL languages.
+    CGFloat minX = fmin(CGRectGetMinX(startSelectionRect), CGRectGetMinX(endSelectionRect));
+    CGFloat maxX = fmax(CGRectGetMaxX(startSelectionRect), CGRectGetMaxX(endSelectionRect));
+    return CGRectMake(minX, minY, maxX - minX, maxY - minY);
+  }
 }

 - (CGRect)caretRectForPosition:(UITextPosition*)position {
--- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.mm
+++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.mm
@@ -1543,24 +1543,264 @@ FLUTTER_ASSERT_ARC
                                  [inputView firstRectForRange:range]));
 }

- (void)testFirstRectForRangeReturnsCorrectSelectionRect {
+- (void)testFirstRectForRangeReturnsCorrectRectOnASingleLineLeftToRight {
  FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
  [inputView setTextInputState:@{@"text" : @"COMPOSING"}];

-  FlutterTextRange* range = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 1)];
-  CGRect testRect = CGRectMake(100, 100, 100, 100);
  [inputView setSelectionRects:@[
    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 0, 100, 100) position:0U],
-    [FlutterTextSelectionRect selectionRectWithRect:testRect position:1U],
-    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, 200, 100, 100) position:2U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 0, 100, 100) position:1U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, 0, 100, 100) position:2U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 0, 100, 100) position:3U],
  ]];
-  XCTAssertTrue(CGRectEqualToRect(testRect, [inputView firstRectForRange:range]));
+  FlutterTextRange* singleRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 1)];
+  XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, 0, 100, 100),
+                                  [inputView firstRectForRange:singleRectRange]));
+
+  FlutterTextRange* multiRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 3)];
+
+  if (@available(iOS 17, *)) {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, 0, 300, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  } else {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, 0, 100, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  }

  [inputView setTextInputState:@{@"text" : @"COM"}];
  FlutterTextRange* rangeOutsideBounds = [FlutterTextRange rangeWithNSRange:NSMakeRange(3, 1)];
  XCTAssertTrue(CGRectEqualToRect(CGRectZero, [inputView firstRectForRange:rangeOutsideBounds]));
 }

+- (void)testFirstRectForRangeReturnsCorrectRectOnASingleLineRightToLeft {
+  FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
+  [inputView setTextInputState:@{@"text" : @"COMPOSING"}];
+
+  [inputView setSelectionRects:@[
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 0, 100, 100) position:0U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, 0, 100, 100) position:1U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 0, 100, 100) position:2U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 0, 100, 100) position:3U],
+  ]];
+  FlutterTextRange* singleRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 1)];
+  XCTAssertTrue(CGRectEqualToRect(CGRectMake(200, 0, 100, 100),
+                                  [inputView firstRectForRange:singleRectRange]));
+
+  FlutterTextRange* multiRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 3)];
+  if (@available(iOS 17, *)) {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(0, 0, 300, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  } else {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(200, 0, 100, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  }
+
+  [inputView setTextInputState:@{@"text" : @"COM"}];
+  FlutterTextRange* rangeOutsideBounds = [FlutterTextRange rangeWithNSRange:NSMakeRange(3, 1)];
+  XCTAssertTrue(CGRectEqualToRect(CGRectZero, [inputView firstRectForRange:rangeOutsideBounds]));
+}
+
+- (void)testFirstRectForRangeReturnsCorrectRectOnMultipleLinesLeftToRight {
+  FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
+  [inputView setTextInputState:@{@"text" : @"COMPOSING"}];
+
+  [inputView setSelectionRects:@[
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 0, 100, 100) position:0U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 0, 100, 100) position:1U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, 0, 100, 100) position:2U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 0, 100, 100) position:3U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 100, 100, 100) position:4U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 100, 100, 100) position:5U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, 100, 100, 100) position:6U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 100, 100, 100) position:7U],
+  ]];
+  FlutterTextRange* singleRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 1)];
+  XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, 0, 100, 100),
+                                  [inputView firstRectForRange:singleRectRange]));
+
+  FlutterTextRange* multiRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 4)];
+
+  if (@available(iOS 17, *)) {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, 0, 300, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  } else {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, 0, 100, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  }
+}
+
+- (void)testFirstRectForRangeReturnsCorrectRectOnMultipleLinesRightToLeft {
+  FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
+  [inputView setTextInputState:@{@"text" : @"COMPOSING"}];
+
+  [inputView setSelectionRects:@[
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 0, 100, 100) position:0U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, 0, 100, 100) position:1U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 0, 100, 100) position:2U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 0, 100, 100) position:3U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 100, 100, 100) position:4U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, 100, 100, 100) position:5U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 100, 100, 100) position:6U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 100, 100, 100) position:7U],
+  ]];
+  FlutterTextRange* singleRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 1)];
+  XCTAssertTrue(CGRectEqualToRect(CGRectMake(200, 0, 100, 100),
+                                  [inputView firstRectForRange:singleRectRange]));
+
+  FlutterTextRange* multiRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 4)];
+  if (@available(iOS 17, *)) {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(0, 0, 300, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  } else {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(200, 0, 100, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  }
+}
+
+- (void)testFirstRectForRangeReturnsCorrectRectOnSingleLineWithVaryingMinYAndMaxYLeftToRight {
+  FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
+  [inputView setTextInputState:@{@"text" : @"COMPOSING"}];
+
+  [inputView setSelectionRects:@[
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 0, 100, 100) position:0U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 10, 100, 80)
+                                           position:1U],  // shorter
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, -10, 100, 120)
+                                           position:2U],  // taller
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 0, 100, 100) position:3U],
+  ]];
+
+  FlutterTextRange* multiRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 3)];
+
+  if (@available(iOS 17, *)) {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, -10, 300, 120),
+                                    [inputView firstRectForRange:multiRectRange]));
+  } else {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, 10, 100, 80),
+                                    [inputView firstRectForRange:multiRectRange]));
+  }
+}
+
+- (void)testFirstRectForRangeReturnsCorrectRectOnSingleLineWithVaryingMinYAndMaxYRightToLeft {
+  FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
+  [inputView setTextInputState:@{@"text" : @"COMPOSING"}];
+
+  [inputView setSelectionRects:@[
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 0, 100, 100) position:0U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, -10, 100, 120)
+                                           position:1U],  // taller
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 10, 100, 80)
+                                           position:2U],  // shorter
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 0, 100, 100) position:3U],
+  ]];
+
+  FlutterTextRange* multiRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 3)];
+
+  if (@available(iOS 17, *)) {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(0, -10, 300, 120),
+                                    [inputView firstRectForRange:multiRectRange]));
+  } else {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(200, -10, 100, 120),
+                                    [inputView firstRectForRange:multiRectRange]));
+  }
+}
+
+- (void)testFirstRectForRangeReturnsCorrectRectWithOverlappingRectsExceedingThresholdLeftToRight {
+  FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
+  [inputView setTextInputState:@{@"text" : @"COMPOSING"}];
+
+  [inputView setSelectionRects:@[
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 0, 100, 100) position:0U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 0, 100, 100) position:1U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, 0, 100, 100) position:2U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 0, 100, 100) position:3U],
+    // y=60 exceeds threshold, so treat it as a new line.
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 60, 100, 100) position:4U],
+  ]];
+
+  FlutterTextRange* multiRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 4)];
+
+  if (@available(iOS 17, *)) {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, 0, 300, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  } else {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, 0, 100, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  }
+}
+
+- (void)testFirstRectForRangeReturnsCorrectRectWithOverlappingRectsExceedingThresholdRightToLeft {
+  FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
+  [inputView setTextInputState:@{@"text" : @"COMPOSING"}];
+
+  [inputView setSelectionRects:@[
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 0, 100, 100) position:0U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, 0, 100, 100) position:1U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 0, 100, 100) position:2U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 0, 100, 100) position:3U],
+    // y=60 exceeds threshold, so treat it as a new line.
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 60, 100, 100) position:4U],
+  ]];
+
+  FlutterTextRange* multiRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 4)];
+
+  if (@available(iOS 17, *)) {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(0, 0, 300, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  } else {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(200, 0, 100, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  }
+}
+
+- (void)testFirstRectForRangeReturnsCorrectRectWithOverlappingRectsWithinThresholdLeftToRight {
+  FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
+  [inputView setTextInputState:@{@"text" : @"COMPOSING"}];
+
+  [inputView setSelectionRects:@[
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 0, 100, 100) position:0U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 0, 100, 100) position:1U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, 0, 100, 100) position:2U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 0, 100, 100) position:3U],
+    // y=40 is within line threshold, so treat it as the same line
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(400, 40, 100, 100) position:4U],
+  ]];
+
+  FlutterTextRange* multiRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 4)];
+
+  if (@available(iOS 17, *)) {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, 0, 400, 140),
+                                    [inputView firstRectForRange:multiRectRange]));
+  } else {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(100, 0, 100, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  }
+}
+
+- (void)testFirstRectForRangeReturnsCorrectRectWithOverlappingRectsWithinThresholdRightToLeft {
+  FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
+  [inputView setTextInputState:@{@"text" : @"COMPOSING"}];
+
+  [inputView setSelectionRects:@[
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(400, 0, 100, 100) position:0U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(300, 0, 100, 100) position:1U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(200, 0, 100, 100) position:2U],
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(100, 0, 100, 100) position:3U],
+    // y=40 is within line threshold, so treat it as the same line
+    [FlutterTextSelectionRect selectionRectWithRect:CGRectMake(0, 40, 100, 100) position:4U],
+  ]];
+
+  FlutterTextRange* multiRectRange = [FlutterTextRange rangeWithNSRange:NSMakeRange(1, 4)];
+
+  if (@available(iOS 17, *)) {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(0, 0, 400, 140),
+                                    [inputView firstRectForRange:multiRectRange]));
+  } else {
+    XCTAssertTrue(CGRectEqualToRect(CGRectMake(300, 0, 100, 100),
+                                    [inputView firstRectForRange:multiRectRange]));
+  }
+}
+
 - (void)testClosestPositionToPoint {
  FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
  [inputView setTextInputState:@{@"text" : @"COMPOSING"}];