From 70cd54c3fd1364ca0c6d3f9f3aa50d6fd28660b8 Mon Sep 17 00:00:00 2001 From: Nils Reichardt Date: Tue, 1 Apr 2025 16:17:20 +0200 Subject: [PATCH] Add `--ignore-timeouts` flag for `flutter test` command (#164437) As in #105913 described, running integration tests of your app often times out. The issue for this is that the `test` package has their own timeout for loading the test suite: https://github.com/dart-lang/test/blob/db8cf091506a67eba1e523215e0e49a0db7cd1fd/pkgs/test_core/lib/src/runner/load_suite.dart#L23-L29 This timeout is not configurable. However, you can bypass this timeout using `--ignore-timeouts` when running `dart test`. This PR adds the `--ignore-timeouts` flag to the `flutter test` command and passes it to the `test` package. Adding the flag would be the easiest fix for #105913. I will later add documentation to the integration test docs, that passing this flag will fix the timeout issue. Otherwise, we would need to make the load test suite timeout configurable in the `test` package and then somehow set it in the `flutter test` command. Fixes #105913 ![Screenshot 2025-03-02 at 00 08 59](https://github.com/user-attachments/assets/af84efe3-2174-4531-919f-bffdf1500430) A screenshot of running `flutter test integration_test --ignore-timeouts` which surpasses the previous timeout of 12 minutes. ## Pre-launch Checklist - [x] I read the [Contributor Guide] and followed the process outlined there for submitting PRs. - [x] I read the [Tree Hygiene] wiki page, which explains my responsibilities. - [x] I read and followed the [Flutter Style Guide], including [Features we expect every widget to implement]. - [x] I signed the [CLA]. - [x] I listed at least one issue that this PR fixes in the description above. - [x] I updated/added relevant documentation (doc comments with `///`). - [x] I added new tests to check the change I am making, or this PR is [test-exempt]. - [x] I followed the [breaking change policy] and added [Data Driven Fixes] where supported. - [x] All existing and new tests are passing. If you need help, consider asking for advice on the #hackers-new channel on [Discord]. [Contributor Guide]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#overview [Tree Hygiene]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md [test-exempt]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#tests [Flutter Style Guide]: https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md [Features we expect every widget to implement]: https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md#features-we-expect-every-widget-to-implement [CLA]: https://cla.developers.google.com/ [flutter/tests]: https://github.com/flutter/tests [breaking change policy]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#handling-breaking-changes [Discord]: https://github.com/flutter/flutter/blob/main/docs/contributing/Chat.md [Data Driven Fixes]: https://github.com/flutter/flutter/blob/main/docs/contributing/Data-driven-Fixes.md --- .../flutter_tools/lib/src/commands/test.dart | 19 +++++++++++++++---- .../flutter_tools/lib/src/test/runner.dart | 4 ++++ .../commands.shard/hermetic/test_test.dart | 5 +++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/packages/flutter_tools/lib/src/commands/test.dart b/packages/flutter_tools/lib/src/commands/test.dart index 1627c190d8..070f7eda9f 100644 --- a/packages/flutter_tools/lib/src/commands/test.dart +++ b/packages/flutter_tools/lib/src/commands/test.dart @@ -273,10 +273,19 @@ class TestCommand extends FlutterCommand with DeviceBasedDevelopmentArtifacts { ..addOption( 'timeout', help: - 'The default test timeout, specified either ' - 'in seconds (e.g. "60s"), ' - 'as a multiplier of the default timeout (e.g. "2x"), ' - 'or as the string "none" to disable the timeout entirely.', + 'The default timeout for individual tests, specified either in ' + 'seconds (e.g. "60s"), as a multiplier of the default test timeout ' + '(e.g. "2x"), or as the string "none" to disable test timeouts ' + 'entirely. This value does not apply to the default test suite ' + 'loading timeout.', + ) + ..addFlag( + 'ignore-timeouts', + help: + 'Ignore all timeouts. Useful when testing a big application ' + 'that requires a longer time to compile (e.g. running integration ' + 'tests for a Flutter app).', + negatable: false, ) ..addFlag( FlutterOptions.kWebWasmFlag, @@ -647,6 +656,7 @@ class TestCommand extends FlutterCommand with DeviceBasedDevelopmentArtifacts { reporter: stringArg('reporter'), fileReporter: stringArg('file-reporter'), timeout: stringArg('timeout'), + ignoreTimeouts: boolArg('ignore-timeouts'), failFast: boolArg('fail-fast'), runSkipped: boolArg('run-skipped'), shardIndex: shardIndex, @@ -675,6 +685,7 @@ class TestCommand extends FlutterCommand with DeviceBasedDevelopmentArtifacts { reporter: stringArg('reporter'), fileReporter: stringArg('file-reporter'), timeout: stringArg('timeout'), + ignoreTimeouts: boolArg('ignore-timeouts'), failFast: boolArg('fail-fast'), runSkipped: boolArg('run-skipped'), shardIndex: shardIndex, diff --git a/packages/flutter_tools/lib/src/test/runner.dart b/packages/flutter_tools/lib/src/test/runner.dart index 944474faec..12a98f0b69 100644 --- a/packages/flutter_tools/lib/src/test/runner.dart +++ b/packages/flutter_tools/lib/src/test/runner.dart @@ -55,6 +55,7 @@ interface class FlutterTestRunner { String? reporter, String? fileReporter, String? timeout, + bool ignoreTimeouts = false, bool failFast = false, bool runSkipped = false, int? shardIndex, @@ -75,6 +76,7 @@ interface class FlutterTestRunner { if (machine) ...['-r', 'json'] else if (reporter != null) ...['-r', reporter], if (fileReporter != null) '--file-reporter=$fileReporter', if (timeout != null) ...['--timeout', timeout], + if (ignoreTimeouts) '--ignore-timeouts', if (concurrency != null) '--concurrency=$concurrency', for (final String name in names) ...['--name', name], for (final String plainName in plainNames) ...['--plain-name', plainName], @@ -588,6 +590,7 @@ class SpawnPlugin extends PlatformPlugin { String? reporter, String? fileReporter, String? timeout, + bool ignoreTimeouts = false, bool failFast = false, bool runSkipped = false, int? shardIndex, @@ -637,6 +640,7 @@ class SpawnPlugin extends PlatformPlugin { if (machine) ...['-r', 'json'] else if (reporter != null) ...['-r', reporter], if (fileReporter != null) '--file-reporter=$fileReporter', if (timeout != null) ...['--timeout', timeout], + if (ignoreTimeouts) '--ignore-timeouts', if (concurrency != null) '--concurrency=$concurrency', for (final String name in names) ...['--name', name], for (final String plainName in plainNames) ...['--plain-name', plainName], diff --git a/packages/flutter_tools/test/commands.shard/hermetic/test_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/test_test.dart index 175ed0d31f..7b87fc3222 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/test_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/test_test.dart @@ -157,6 +157,7 @@ dev_dependencies: expect(fakePackageTest.lastArgs, isNot(contains('compact'))); expect(fakePackageTest.lastArgs, isNot(contains('--timeout'))); expect(fakePackageTest.lastArgs, isNot(contains('30s'))); + expect(fakePackageTest.lastArgs, isNot(contains('--ignore-timeouts'))); expect(fakePackageTest.lastArgs, isNot(contains('--concurrency'))); }, overrides: { @@ -583,6 +584,7 @@ dev_dependencies: '--reporter=compact', '--file-reporter=json:reports/tests.json', '--timeout=100', + '--ignore-timeouts', '--concurrency=3', '--name=name1', '--plain-name=name2', @@ -616,6 +618,7 @@ const List packageTestArgs = [ '--file-reporter=json:reports/tests.json', '--timeout', '100', + '--ignore-timeouts', '--concurrency=3', '--name', 'name1', @@ -1562,6 +1565,7 @@ class FakeFlutterTestRunner implements FlutterTestRunner { String? reporter, String? fileReporter, String? timeout, + bool ignoreTimeouts = false, bool failFast = false, bool runSkipped = false, int? shardIndex, @@ -1611,6 +1615,7 @@ class FakeFlutterTestRunner implements FlutterTestRunner { String? reporter, String? fileReporter, String? timeout, + bool ignoreTimeouts = false, bool failFast = false, bool runSkipped = false, int? shardIndex,