diff --git a/dev/automated_tests/integration_test/trivial_widget_expectation.txt b/dev/automated_tests/integration_test/trivial_widget_expectation.txt index ae9e629a1a..cda62d712c 100644 --- a/dev/automated_tests/integration_test/trivial_widget_expectation.txt +++ b/dev/automated_tests/integration_test/trivial_widget_expectation.txt @@ -1,2 +1 @@ -<> [0-9]+:[0-9]+ [+]1: All tests passed! diff --git a/packages/flutter_tools/lib/src/commands/test.dart b/packages/flutter_tools/lib/src/commands/test.dart index aa301ba937..e4993a10c5 100644 --- a/packages/flutter_tools/lib/src/commands/test.dart +++ b/packages/flutter_tools/lib/src/commands/test.dart @@ -401,6 +401,10 @@ class TestCommand extends FlutterCommand with DeviceBasedDevelopmentArtifacts { if (_isIntegrationTest) { integrationTestDevice = await findTargetDevice(); + // Disable reporting of test results to native test frameworks. This isn't + // needed as the Flutter Tool will be responsible for reporting results. + buildInfo.dartDefines.add('INTEGRATION_TEST_SHOULD_REPORT_RESULTS_TO_NATIVE=false'); + if (integrationTestDevice == null) { throwToolExit( 'No devices are connected. ' diff --git a/packages/flutter_tools/test/commands.shard/hermetic/test_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/test_test.dart index 5070661b1c..6bc5f5caef 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/test_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/test_test.dart @@ -582,6 +582,30 @@ dev_dependencies: FakeDevice('ephemeral', 'ephemeral', ephemeral: true, isSupported: true, type: PlatformType.web), ]), }); + + testUsingContext('Integration tests set the correct dart-defines', () async { + final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); + + final TestCommand testCommand = TestCommand(testRunner: testRunner); + final CommandRunner commandRunner = createTestCommandRunner(testCommand); + + await commandRunner.run(const [ + 'test', + '--no-pub', + 'integration_test', + ]); + + expect( + testRunner.lastDebuggingOptionsValue.buildInfo.dartDefines, + contains('INTEGRATION_TEST_SHOULD_REPORT_RESULTS_TO_NATIVE=false'), + ); + }, overrides: { + FileSystem: () => fs, + ProcessManager: () => FakeProcessManager.any(), + DeviceManager: () => _FakeDeviceManager([ + FakeDevice('ephemeral', 'ephemeral', ephemeral: true, isSupported: true, type: PlatformType.android), + ]), + }); } class FakeFlutterTestRunner implements FlutterTestRunner { @@ -589,6 +613,7 @@ class FakeFlutterTestRunner implements FlutterTestRunner { int exitCode; bool lastEnableObservatoryValue; + DebuggingOptions lastDebuggingOptionsValue; @override Future runTests( @@ -626,6 +651,7 @@ class FakeFlutterTestRunner implements FlutterTestRunner { String integrationTestUserIdentifier, }) async { lastEnableObservatoryValue = enableObservatory; + lastDebuggingOptionsValue = debuggingOptions; return exitCode; } } diff --git a/packages/integration_test/lib/integration_test.dart b/packages/integration_test/lib/integration_test.dart index ac3c1029d6..a24111906c 100644 --- a/packages/integration_test/lib/integration_test.dart +++ b/packages/integration_test/lib/integration_test.dart @@ -20,24 +20,37 @@ import 'common.dart'; const String _success = 'success'; +/// Whether results should be reported to the native side over the method +/// channel. +/// +/// This is enabled by default for use by native test frameworks like Android +/// instrumentation or XCTest. When running with the Flutter Tool through +/// `flutter test integration_test` though, it will be disabled as the Flutter +/// tool will be responsible for collection of test results. +const bool _shouldReportResultsToNative = bool.fromEnvironment( + 'INTEGRATION_TEST_SHOULD_REPORT_RESULTS_TO_NATIVE', + defaultValue: true, +); + /// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results /// on a channel to adapt them to native instrumentation test format. class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding implements IntegrationTestResults { /// Sets up a listener to report that the tests are finished when everything is /// torn down. IntegrationTestWidgetsFlutterBinding() { - // TODO(jackson): Report test results as they arrive tearDownAll(() async { + if (!_allTestsPassed.isCompleted) { + _allTestsPassed.complete(true); + } + callbackManager.cleanup(); + + // TODO(jiahaog): Print the message directing users to run with + // `flutter test` when Web is supported. + if (!_shouldReportResultsToNative || kIsWeb) { + return; + } + try { - // For web integration tests we are not using the - // `plugins.flutter.io/integration_test`. Mark the tests as complete - // before invoking the channel. - if (kIsWeb) { - if (!_allTestsPassed.isCompleted) { - _allTestsPassed.complete(true); - } - } - callbackManager.cleanup(); await _channel.invokeMethod( 'allTestsFinished', { @@ -50,15 +63,22 @@ class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding }, ); } on MissingPluginException { - print('Warning: integration_test test plugin was not detected.'); - } - if (!_allTestsPassed.isCompleted) { - _allTestsPassed.complete(true); + print(r''' +Warning: integration_test plugin was not detected. + +If you're running the tests with `flutter drive`, please make sure your tests +are in the `integration_test/` directory of your package and use +`flutter test $path_to_test` to run it instead. + +If you're running the tests with Android instrumentation or XCTest, this means +that you are not capturing test results properly! See the following link for +how to set up the integration_test plugin: + +https://flutter.dev/docs/testing/integration-tests#testing-on-firebase-test-lab +'''); } }); - // TODO(jackson): Report the results individually instead of all at once - // See https://github.com/flutter/flutter/issues/38985 final TestExceptionReporter oldTestExceptionReporter = reportTestException; reportTestException = (FlutterErrorDetails details, String testDescription) {