From af5eb3b987e1749692ff226be18b087006908e4e Mon Sep 17 00:00:00 2001 From: Jia Hao Date: Fri, 13 Nov 2020 19:57:15 +0800 Subject: [PATCH] [integration_test] Add a `run` method for proper reporting of test results (#70075) --- packages/integration_test/README.md | 4 +- .../integration_test/_example_test_web.dart | 2 - .../integration_test/_extended_test_io.dart | 3 - .../integration_test/example_test.dart | 5 +- .../integration_test/extended_test.dart | 5 +- packages/integration_test/lib/common.dart | 39 +++- .../lib/integration_test.dart | 180 +++++++++++++----- .../integration_test/lib/src/constants.dart | 6 + .../integration_test/lib/src/reporter.dart | 55 ++++++ packages/integration_test/pubspec.yaml | 22 ++- .../test/binding_fail_test.dart | 79 -------- .../test/{ => reporter}/data/README.md | 0 .../{ => reporter}/data/fail_test_script.dart | 17 +- .../{ => reporter}/data/pass_test_script.dart | 12 +- .../data/pass_then_fail_test_script.dart | 14 +- .../test/reporter/legacy_reporter_test.dart | 77 ++++++++ .../reporter/multiple_tests_fail_test.dart | 29 +++ .../reporter/multiple_tests_pass_test.dart | 29 +++ .../test/reporter/pass_then_fail_test.dart | 29 +++ .../integration_test/test/reporter/utils.dart | 44 +++++ 20 files changed, 481 insertions(+), 170 deletions(-) create mode 100644 packages/integration_test/lib/src/constants.dart create mode 100644 packages/integration_test/lib/src/reporter.dart delete mode 100644 packages/integration_test/test/binding_fail_test.dart rename packages/integration_test/test/{ => reporter}/data/README.md (100%) rename packages/integration_test/test/{ => reporter}/data/fail_test_script.dart (59%) rename packages/integration_test/test/{ => reporter}/data/pass_test_script.dart (66%) rename packages/integration_test/test/{ => reporter}/data/pass_then_fail_test_script.dart (63%) create mode 100644 packages/integration_test/test/reporter/legacy_reporter_test.dart create mode 100644 packages/integration_test/test/reporter/multiple_tests_fail_test.dart create mode 100644 packages/integration_test/test/reporter/multiple_tests_pass_test.dart create mode 100644 packages/integration_test/test/reporter/pass_then_fail_test.dart create mode 100644 packages/integration_test/test/reporter/utils.dart diff --git a/packages/integration_test/README.md b/packages/integration_test/README.md index be08a722bc..0eef2b2fb4 100644 --- a/packages/integration_test/README.md +++ b/packages/integration_test/README.md @@ -18,9 +18,9 @@ assertions. import 'package:flutter_test/flutter_test.dart'; import 'package:integration_test/integration_test.dart'; -void main() { - IntegrationTestWidgetsFlutterBinding.ensureInitialized(); +void main() => run(_testMain); +void _testMain() { testWidgets("failing test example", (WidgetTester tester) async { expect(2 + 2, equals(5)); }); diff --git a/packages/integration_test/example/integration_test/_example_test_web.dart b/packages/integration_test/example/integration_test/_example_test_web.dart index cd5e986ded..856559a846 100644 --- a/packages/integration_test/example/integration_test/_example_test_web.dart +++ b/packages/integration_test/example/integration_test/_example_test_web.dart @@ -12,12 +12,10 @@ import 'dart:html' as html; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; -import 'package:integration_test/integration_test.dart'; import 'package:integration_test_example/main.dart' as app; void main() { - IntegrationTestWidgetsFlutterBinding.ensureInitialized(); testWidgets('verify text', (WidgetTester tester) async { // Build our app and trigger a frame. app.main(); diff --git a/packages/integration_test/example/integration_test/_extended_test_io.dart b/packages/integration_test/example/integration_test/_extended_test_io.dart index f67c94ab4e..359135fca2 100644 --- a/packages/integration_test/example/integration_test/_extended_test_io.dart +++ b/packages/integration_test/example/integration_test/_extended_test_io.dart @@ -12,13 +12,10 @@ import 'dart:io' show Platform; import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; -import 'package:integration_test/integration_test.dart'; import 'package:integration_test_example/main.dart' as app; void main() { - IntegrationTestWidgetsFlutterBinding.ensureInitialized(); - testWidgets('verify text', (WidgetTester tester) async { // Build our app and trigger a frame. app.main(); diff --git a/packages/integration_test/example/integration_test/example_test.dart b/packages/integration_test/example/integration_test/example_test.dart index 8a9c97701f..6628ff2b41 100644 --- a/packages/integration_test/example/integration_test/example_test.dart +++ b/packages/integration_test/example/integration_test/example_test.dart @@ -14,7 +14,4 @@ import 'package:integration_test/integration_test.dart'; import '_example_test_io.dart' if (dart.library.html) '_example_test_web.dart' as tests; -void main() { - IntegrationTestWidgetsFlutterBinding.ensureInitialized(); - tests.main(); -} +void main() => run(tests.main); diff --git a/packages/integration_test/example/integration_test/extended_test.dart b/packages/integration_test/example/integration_test/extended_test.dart index d43296dd1b..7acb32ad36 100644 --- a/packages/integration_test/example/integration_test/extended_test.dart +++ b/packages/integration_test/example/integration_test/extended_test.dart @@ -17,7 +17,4 @@ import 'package:integration_test/integration_test.dart'; import '_extended_test_io.dart' if (dart.library.html) '_extended_test_web.dart' as tests; -void main() { - IntegrationTestWidgetsFlutterBinding.ensureInitialized(); - tests.main(); -} +void main() => run(tests.main); diff --git a/packages/integration_test/lib/common.dart b/packages/integration_test/lib/common.dart index 23a9081e0e..44c3869798 100644 --- a/packages/integration_test/lib/common.dart +++ b/packages/integration_test/lib/common.dart @@ -106,21 +106,48 @@ class Response { } } -/// Representing a failure includes the method name and the failure details. -class Failure { - /// Constructor requiring all fields during initialization. - Failure(this.methodName, this.details); +/// Represents the result of running a test. +class TestResult { + TestResult._(this.methodName); /// The name of the test method which failed. final String methodName; +} - /// The details of the failure such as stack trace. - final String details; +/// Represents successful execution of a test. +class Success extends TestResult { + /// Constructor requiring all fields during initialization. + Success(String methodName) : super._(methodName); +} + +/// Represents a test failure. +class Failure extends TestResult { + /// Constructor requiring all fields during initialization. + /// + /// If [error] is passed, [errors] will be ignored. + Failure(String methodName, String details, { + Object error, + List errors, + }) : + errors = error != null + ? [AsyncError(error, StackTrace.fromString(details))] + : errors ?? [], + super._(methodName); + + /// Errors that were thrown during the test. + final List errors; + + /// The first error that was thrown during the test. + Object get error => errors.isEmpty ? null : errors.first.error; + + /// The details of the first failure such as stack trace. + String get details => errors.isEmpty ? null : errors.first.stackTrace.toString(); /// Serializes the object to JSON. String toJson() { return json.encode({ 'methodName': methodName, + 'error': error.toString(), 'details': details, }); } diff --git a/packages/integration_test/lib/integration_test.dart b/packages/integration_test/lib/integration_test.dart index 2d722ae3ac..5306e8f461 100644 --- a/packages/integration_test/lib/integration_test.dart +++ b/packages/integration_test/lib/integration_test.dart @@ -6,70 +6,157 @@ import 'dart:async'; import 'dart:developer' as developer; import 'dart:ui'; -import 'package:flutter/rendering.dart'; -import 'package:flutter_test/flutter_test.dart'; import 'package:flutter/foundation.dart'; +import 'package:flutter/rendering.dart'; import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; +// ignore: implementation_imports +import 'package:test_core/src/direct_run.dart'; +// ignore: implementation_imports +import 'package:test_core/src/runner/engine.dart'; import 'package:vm_service/vm_service.dart' as vm; import 'package:vm_service/vm_service_io.dart' as vm_io; -import '_callback_io.dart' if (dart.library.html) '_callback_web.dart' as driver_actions; +import '_callback_io.dart' if (dart.library.html) '_callback_web.dart' + as driver_actions; import '_extension_io.dart' if (dart.library.html) '_extension_web.dart'; import 'common.dart'; +import 'src/constants.dart'; +import 'src/reporter.dart'; -const String _success = 'success'; +/// Toggles the legacy reporting mechansim where results are only collected +/// for [testWidgets]. +/// +/// If [run] is called, this will be disabled. +bool _isUsingLegacyReporting = true; + +/// Executes a block that contains tests. +/// +/// Example Usage: +/// ``` +/// import 'package:flutter_test/flutter_test.dart'; +/// import 'package:integration_test/integration_test.dart'; +/// +/// void main() => run(_testMain); +/// +/// void _testMain() { +/// test('A test', () { +/// expect(true, true); +/// }); +/// } +/// ``` +/// +/// If not explicitly passed, the default [reporter] will send results over the +/// platform channel to native. +Future run( + FutureOr Function() testMain, { + Reporter reporter = const _ReporterImpl(), +}) async { + _isUsingLegacyReporting = false; + final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding; + + // Pipe detailed exceptions within [testWidgets] to `package:test`. + reportTestException = (FlutterErrorDetails details, String testDescription) { + registerException('Test $testDescription failed: $details'); + }; + + final Completer> resultsCompleter = Completer>(); + + await directRunTests( + testMain, + reporterFactory: (Engine engine) => ResultReporter(engine, resultsCompleter), + ); + + final List results = await resultsCompleter.future; + + binding._updateTestResultState({ + for (final TestResult result in results) + result.methodName: result, + }); + await reporter.report(results); +} + +/// Abstract interface for a result reporter. +abstract class Reporter { + /// Reports test results. + /// + /// This method will be called at the end of [run] with the [results] of + /// running the test suite. + Future report(List results); +} + +/// Default implementation of the reporter that sends results over to the +/// platform side. +class _ReporterImpl implements Reporter { + const _ReporterImpl(); + + @override + Future report( + List results, + ) async { + try { + await IntegrationTestWidgetsFlutterBinding._channel.invokeMethod( + 'allTestsFinished', + { + 'results': { + for (final TestResult result in results) + result.methodName: result is Failure + ? _formatFailureForPlatform(result) + : success + } + }, + ); + } on MissingPluginException { + print('Warning: integration_test test plugin was not detected.'); + } + } +} + +String _formatFailureForPlatform(Failure failure) => '${failure.error} ${failure.details}'; /// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results /// on a channel to adapt them to native instrumentation test format. -class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding implements IntegrationTestResults { - /// Sets up a listener to report that the tests are finished when everything is - /// torn down. +class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding + implements IntegrationTestResults { + /// If [run] is not used, sets up a listener to report that the tests are + /// finished when everything is torn down. + /// + /// This functionality is deprecated – clients are expected to use [run] to + /// execute their tests instead. IntegrationTestWidgetsFlutterBinding() { - // TODO(jackson): Report test results as they arrive + if (!_isUsingLegacyReporting) { + // TODO(jiahaog): Point users to use the CLI https://github.com/flutter/flutter/issues/66264. + print('Using the legacy test result reporter, which will not catch all ' + 'errors thrown in declared tests. Consider wrapping tests with ' + 'https://api.flutter.dev/flutter/integration_test/run.html instead.'); + return; + } + tearDownAll(() async { - try { - // For web integration tests we are not using the - // `plugins.flutter.io/integration_test`. Mark the tests as complete - // before invoking the channel. - if (kIsWeb) { - if (!_allTestsPassed.isCompleted) { - _allTestsPassed.complete(true); - } - } - callbackManager.cleanup(); - await _channel.invokeMethod( - 'allTestsFinished', - { - 'results': results.map((String name, Object result) { - if (result is Failure) { - return MapEntry(name, result.details); - } - return MapEntry(name, result); - }) - }, - ); - } on MissingPluginException { - print('Warning: integration_test test plugin was not detected.'); - } - if (!_allTestsPassed.isCompleted) { - _allTestsPassed.complete(true); - } + _updateTestResultState(results); + await const _ReporterImpl().report(results.values.toList()); }); - // TODO(jackson): Report the results individually instead of all at once - // See https://github.com/flutter/flutter/issues/38985 final TestExceptionReporter oldTestExceptionReporter = reportTestException; - reportTestException = - (FlutterErrorDetails details, String testDescription) { - results[testDescription] = Failure(testDescription, details.toString()); - if (!_allTestsPassed.isCompleted) { - _allTestsPassed.complete(false); - } + reportTestException = (FlutterErrorDetails details, String testDescription) { + results[testDescription] = Failure( + testDescription, + details.toString(), + error: details.exception, + ); oldTestExceptionReporter(details, testDescription); }; } + void _updateTestResultState(Map results) { + this.results = results; + print('Test execution completed: $results'); + + _allTestsPassed.complete(!results.values.any((TestResult result) => result is Failure)); + callbackManager.cleanup(); + } + @override bool get overrideHttpClient => false; @@ -131,11 +218,8 @@ class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding MethodChannel('plugins.flutter.io/integration_test'); /// Test results that will be populated after the tests have completed. - /// - /// Keys are the test descriptions, and values are either [_success] or - /// a [Failure]. @visibleForTesting - Map results = {}; + Map results = {}; List get _failures => results.values.whereType().toList(); @@ -191,7 +275,7 @@ class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding description: description, timeout: timeout, ); - results[description] ??= _success; + results[description] ??= Success(description); } vm.VmService _vmService; diff --git a/packages/integration_test/lib/src/constants.dart b/packages/integration_test/lib/src/constants.dart new file mode 100644 index 0000000000..bb9dcdb7c4 --- /dev/null +++ b/packages/integration_test/lib/src/constants.dart @@ -0,0 +1,6 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// Represents a successful test. +const String success = 'success'; diff --git a/packages/integration_test/lib/src/reporter.dart b/packages/integration_test/lib/src/reporter.dart new file mode 100644 index 0000000000..1343c0a99f --- /dev/null +++ b/packages/integration_test/lib/src/reporter.dart @@ -0,0 +1,55 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +// ignore: implementation_imports +import 'package:test_api/src/backend/live_test.dart'; +// ignore: implementation_imports +import 'package:test_core/src/runner/engine.dart'; +// ignore: implementation_imports +import 'package:test_core/src/runner/reporter.dart'; + +import '../common.dart'; +import 'constants.dart'; + +/// A reporter that plugs into [directRunTests] from `package:test_core`. +class ResultReporter implements Reporter { + /// When the [_engine] has completed execution of tests, [_resultsCompleter] + /// will be completed with the test results. + ResultReporter(this._engine, this._resultsCompleter) { + _subscriptions.add(_engine.success.asStream().listen(_onDone)); + } + final Engine _engine; + final Completer> _resultsCompleter; + + final Set> _subscriptions = >{}; + + void _onDone(bool _) { + _cancel(); + final List results = [ + for (final LiveTest liveTest in _engine.liveTests) + liveTest.state.result.name == success + ? Success(liveTest.test.name) + : Failure( + liveTest.test.name, + null, + errors: liveTest.errors, + ) + ]; + _resultsCompleter.complete(results); + } + + void _cancel() { + for (final StreamSubscription subscription in _subscriptions) { + subscription.cancel(); + } + _subscriptions.clear(); + } + + @override + void pause() {} + @override + void resume() {} +} diff --git a/packages/integration_test/pubspec.yaml b/packages/integration_test/pubspec.yaml index 130aaa5eb0..4a5c243ee0 100644 --- a/packages/integration_test/pubspec.yaml +++ b/packages/integration_test/pubspec.yaml @@ -15,22 +15,40 @@ dependencies: flutter_test: sdk: flutter path: 1.8.0-nullsafety.3 + test_core: 0.3.12-nullsafety.9 vm_service: 5.2.0 + _fe_analyzer_shared: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + analyzer: 0.39.17 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" archive: 2.0.13 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" args: 1.6.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" async: 2.5.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" boolean_selector: 2.1.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" characters: 1.1.0-nullsafety.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" charcode: 1.2.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + cli_util: 0.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" clock: 1.1.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" collection: 1.15.0-nullsafety.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" convert: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + coverage: 0.14.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" crypto: 2.1.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + csslib: 0.16.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" fake_async: 1.2.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" file: 6.0.0-nullsafety.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + glob: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + html: 0.14.0+4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + io: 0.3.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + js: 0.6.3-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + logging: 0.11.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" matcher: 0.12.10-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" meta: 1.3.0-nullsafety.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + node_interop: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + node_io: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + package_config: 1.9.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + pool: 1.5.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + pub_semver: 1.4.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + source_map_stack_trace: 2.1.0-nullsafety.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + source_maps: 0.10.10-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" source_span: 1.8.0-nullsafety.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" stack_trace: 1.10.0-nullsafety.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" stream_channel: 2.1.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" @@ -40,7 +58,9 @@ dependencies: test_api: 0.2.19-nullsafety.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" typed_data: 1.3.0-nullsafety.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" vector_math: 2.1.0-nullsafety.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + watcher: 0.9.7+15 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" webdriver: 2.1.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + yaml: 2.2.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" dev_dependencies: pedantic: 1.10.0-nullsafety.3 @@ -55,4 +75,4 @@ flutter: ios: pluginClass: IntegrationTestPlugin -# PUBSPEC CHECKSUM: f9bc +# PUBSPEC CHECKSUM: a5dd diff --git a/packages/integration_test/test/binding_fail_test.dart b/packages/integration_test/test/binding_fail_test.dart deleted file mode 100644 index 622c970c3b..0000000000 --- a/packages/integration_test/test/binding_fail_test.dart +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; - -import 'package:flutter_test/flutter_test.dart'; -import 'package:path/path.dart' as path; - -final String bat = Platform.isWindows ? '.bat' : ''; -final String _flutterBin = path.join(Directory.current.parent.parent.parent.path, 'bin', 'flutter$bat'); -const String _integrationResultsPrefix = - 'IntegrationTestWidgetsFlutterBinding test results:'; -const String _failureExcerpt = r'Expected: \n Actual: '; - -Future main() async { - group('Integration binding result', () { - test('when multiple tests pass', () async { - final Map results = await _runTest(path.join('test', 'data', 'pass_test_script.dart')); - - expect( - results, - equals({ - 'passing test 1': 'success', - 'passing test 2': 'success', - })); - }); - - test('when multiple tests fail', () async { - final Map results = await _runTest(path.join('test', 'data', 'fail_test_script.dart')); - - expect(results, hasLength(2)); - expect(results, containsPair('failing test 1', contains(_failureExcerpt))); - expect(results, containsPair('failing test 2', contains(_failureExcerpt))); - }); - - test('when one test passes, then another fails', () async { - final Map results = await _runTest(path.join('test', 'data', 'pass_then_fail_test_script.dart')); - - expect(results, hasLength(2)); - expect(results, containsPair('passing test', equals('success'))); - expect(results, containsPair('failing test', contains(_failureExcerpt))); - }); - }); -} - -/// Runs a test script and returns the [IntegrationTestWidgetsFlutterBinding.result]. -/// -/// [scriptPath] is relative to the package root. -Future> _runTest(String scriptPath) async { - final Process process = - await Process.start(_flutterBin, ['test', '--machine', scriptPath]); - - /// In the test [tearDownAll] block, the test results are encoded into JSON and - /// are printed with the [_integrationResultsPrefix] prefix. - /// - /// See the following for the test event spec which we parse the printed lines - /// out of: https://github.com/dart-lang/test/blob/master/pkgs/test/doc/json_reporter.md - final String testResults = (await process.stdout - .transform(utf8.decoder) - .expand((String text) => text.split('\n')) - .map((String line) { - try { - return jsonDecode(line) as Map; - } on FormatException { - // Only interested in test events which are JSON. - } - }) - .where((Map testEvent) => - testEvent != null && testEvent['type'] == 'print') - .map((Map printEvent) => printEvent['message'] as String) - .firstWhere((String message) => - message.startsWith(_integrationResultsPrefix))) - .replaceAll(_integrationResultsPrefix, ''); - - return jsonDecode(testResults) as Map; -} diff --git a/packages/integration_test/test/data/README.md b/packages/integration_test/test/reporter/data/README.md similarity index 100% rename from packages/integration_test/test/data/README.md rename to packages/integration_test/test/reporter/data/README.md diff --git a/packages/integration_test/test/data/fail_test_script.dart b/packages/integration_test/test/reporter/data/fail_test_script.dart similarity index 59% rename from packages/integration_test/test/data/fail_test_script.dart rename to packages/integration_test/test/reporter/data/fail_test_script.dart index 58db5719dd..c1b073baec 100644 --- a/packages/integration_test/test/data/fail_test_script.dart +++ b/packages/integration_test/test/reporter/data/fail_test_script.dart @@ -2,23 +2,24 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:convert'; - import 'package:integration_test/integration_test.dart'; import 'package:flutter_test/flutter_test.dart'; -Future main() async { +import '../utils.dart'; + +void main() { final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding; - testWidgets('failing test 1', (WidgetTester tester) async { - expect(true, false); + testWidgets('Failing test 1', (WidgetTester tester) async { + expect(false, true); }); - testWidgets('failing test 2', (WidgetTester tester) async { - expect(true, false); + testWidgets('Failing test 2', (WidgetTester tester) async { + expect(false, true); }); tearDownAll(() { - print('IntegrationTestWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}'); + print( + 'IntegrationTestWidgetsFlutterBinding test results: ${testResultsToJson(binding.results)}'); }); } diff --git a/packages/integration_test/test/data/pass_test_script.dart b/packages/integration_test/test/reporter/data/pass_test_script.dart similarity index 66% rename from packages/integration_test/test/data/pass_test_script.dart rename to packages/integration_test/test/reporter/data/pass_test_script.dart index b0d3c95cac..5264d54d79 100644 --- a/packages/integration_test/test/data/pass_test_script.dart +++ b/packages/integration_test/test/reporter/data/pass_test_script.dart @@ -2,24 +2,24 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:convert'; - import 'package:integration_test/integration_test.dart'; import 'package:flutter_test/flutter_test.dart'; -Future main() async { +import '../utils.dart'; + +void main() { final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding; - testWidgets('passing test 1', (WidgetTester tester) async { + testWidgets('Passing test 1', (WidgetTester tester) async { expect(true, true); }); - testWidgets('passing test 2', (WidgetTester tester) async { + testWidgets('Passing test 2', (WidgetTester tester) async { expect(true, true); }); tearDownAll(() { print( - 'IntegrationTestWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}'); + 'IntegrationTestWidgetsFlutterBinding test results: ${testResultsToJson(binding.results)}'); }); } diff --git a/packages/integration_test/test/data/pass_then_fail_test_script.dart b/packages/integration_test/test/reporter/data/pass_then_fail_test_script.dart similarity index 63% rename from packages/integration_test/test/data/pass_then_fail_test_script.dart rename to packages/integration_test/test/reporter/data/pass_then_fail_test_script.dart index 6597ac764c..12ec802683 100644 --- a/packages/integration_test/test/data/pass_then_fail_test_script.dart +++ b/packages/integration_test/test/reporter/data/pass_then_fail_test_script.dart @@ -2,24 +2,24 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:convert'; - import 'package:integration_test/integration_test.dart'; import 'package:flutter_test/flutter_test.dart'; -Future main() async { +import '../utils.dart'; + +void main() { final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding; - testWidgets('passing test', (WidgetTester tester) async { + testWidgets('Passing test', (WidgetTester tester) async { expect(true, true); }); - testWidgets('failing test', (WidgetTester tester) async { - expect(true, false); + testWidgets('Failing test', (WidgetTester tester) async { + expect(false, true); }); tearDownAll(() { print( - 'IntegrationTestWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}'); + 'IntegrationTestWidgetsFlutterBinding test results: ${testResultsToJson(binding.results)}'); }); } diff --git a/packages/integration_test/test/reporter/legacy_reporter_test.dart b/packages/integration_test/test/reporter/legacy_reporter_test.dart new file mode 100644 index 0000000000..b88059c19b --- /dev/null +++ b/packages/integration_test/test/reporter/legacy_reporter_test.dart @@ -0,0 +1,77 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/src/constants.dart'; + +import 'utils.dart'; + +// Assumes that the flutter command is in `$PATH`. +const String _flutterBin = 'flutter'; +const String _integrationResultsPrefix = 'IntegrationTestWidgetsFlutterBinding test results:'; + +Future main() async { + test('When multiple tests pass', () async { + final Map results = await _runTest('test/reporter/data/pass_test_script.dart'); + + expect(results, hasLength(2)); + expect(results, containsPair('Passing test 1', _isSuccess)); + expect(results, containsPair('Passing test 2', _isSuccess)); + }); + + test('When multiple tests fail', () async { + final Map results = await _runTest('test/reporter/data/fail_test_script.dart'); + + expect(results, hasLength(2)); + expect(results, containsPair('Failing test 1', _isSerializedFailure)); + expect(results, containsPair('Failing test 2', _isSerializedFailure)); + }); + + test('When one test passes, then another fails', () async { + final Map results = await _runTest('test/reporter/data/pass_then_fail_test_script.dart'); + + expect(results, hasLength(2)); + expect(results, containsPair('Passing test', _isSuccess)); + expect(results, containsPair('Failing test', _isSerializedFailure)); + }); +} + +/// Runs a test script and returns the [IntegrationTestWidgetsFlutterBinding.result]. +/// +/// [scriptPath] is relative to the package root. +Future> _runTest(String scriptPath) async { + final Process process = await Process.start(_flutterBin, ['test', '--machine', scriptPath]); + + /// In the test [tearDownAll] block, the test results are encoded into JSON and + /// are printed with the [_integrationResultsPrefix] prefix. + /// + /// See the following for the test event spec which we parse the printed lines + /// out of: https://github.com/dart-lang/test/blob/master/pkgs/test/doc/json_reporter.md + final String testResults = (await process.stdout + .transform(utf8.decoder) + .expand((String text) => text.split('\n')) + .map((String line) { + try { + return jsonDecode(line); + } on FormatException { + // Only interested in test events which are JSON. + } + }) + .where((dynamic testEvent) => + testEvent != null && testEvent['type'] == 'print') + .map((dynamic printEvent) => printEvent['message'] as String) + .firstWhere((String message) => + message.startsWith(_integrationResultsPrefix))) + .replaceAll(_integrationResultsPrefix, ''); + + return jsonDecode(testResults) as Map; +} + +bool _isSuccess(Object object) => object == success; + +bool _isSerializedFailure(dynamic object) => object.toString().contains(failureExcerpt); diff --git a/packages/integration_test/test/reporter/multiple_tests_fail_test.dart b/packages/integration_test/test/reporter/multiple_tests_fail_test.dart new file mode 100644 index 0000000000..c5be77a28b --- /dev/null +++ b/packages/integration_test/test/reporter/multiple_tests_fail_test.dart @@ -0,0 +1,29 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/common.dart'; + +import 'utils.dart'; + +void main() { + test('When multiple tests fail', () async { + final List results = await runAndCollectResults(_testMain); + + expect(results, [ + isFailure('Failing testWidgets()'), + isFailure('Failing test()') + ]); + }); +} + +void _testMain() { + testWidgets('Failing testWidgets()', (WidgetTester tester) async { + expect(false, true); + }); + + test('Failing test()', () { + expect(false, true); + }); +} diff --git a/packages/integration_test/test/reporter/multiple_tests_pass_test.dart b/packages/integration_test/test/reporter/multiple_tests_pass_test.dart new file mode 100644 index 0000000000..6bff5c7478 --- /dev/null +++ b/packages/integration_test/test/reporter/multiple_tests_pass_test.dart @@ -0,0 +1,29 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/common.dart'; + +import 'utils.dart'; + +void main() { + test('When multiple tests pass', () async { + final List results = await runAndCollectResults(_testMain); + + expect(results, [ + isSuccess('Passing testWidgets()'), + isSuccess('Passing test()') + ]); + }); +} + +void _testMain() { + testWidgets('Passing testWidgets()', (WidgetTester tester) async { + expect(true, true); + }); + + test('Passing test()', () { + expect(true, true); + }); +} diff --git a/packages/integration_test/test/reporter/pass_then_fail_test.dart b/packages/integration_test/test/reporter/pass_then_fail_test.dart new file mode 100644 index 0000000000..81834b859a --- /dev/null +++ b/packages/integration_test/test/reporter/pass_then_fail_test.dart @@ -0,0 +1,29 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/common.dart'; + +import 'utils.dart'; + +void main() { + test('when one test passes, then another fails', () async { + final List results = await runAndCollectResults(_testMain); + + expect(results, [ + isSuccess('Passing test'), + isFailure('Failing test') + ]); + }); +} + +void _testMain() { + testWidgets('Passing test', (WidgetTester tester) async { + expect(true, true); + }); + + testWidgets('Failing test', (WidgetTester tester) async { + expect(false, true); + }); +} diff --git a/packages/integration_test/test/reporter/utils.dart b/packages/integration_test/test/reporter/utils.dart new file mode 100644 index 0000000000..793b525e86 --- /dev/null +++ b/packages/integration_test/test/reporter/utils.dart @@ -0,0 +1,44 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/common.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:integration_test/src/constants.dart'; + +const String failureExcerpt = 'Expected: '; + +dynamic isSuccess(String methodName) => isA() + .having((Success s) => s.methodName, 'methodName', methodName); + +dynamic isFailure(String methodName) => isA() + .having((Failure e) => e.methodName, 'methodName', methodName) + .having((Failure e) => e.error.toString(), 'error', contains(failureExcerpt)); + + +Future> runAndCollectResults( + FutureOr Function() testMain, +) async { + final _TestReporter reporter = _TestReporter(); + await run(testMain, reporter: reporter); + return reporter.results; +} + +class _TestReporter implements Reporter { + final Completer> _resultsCompleter = Completer>(); + Future> get results => _resultsCompleter.future; + + @override + Future report(List results) async => _resultsCompleter.complete(results); +} + +String testResultsToJson(Map results) { + return jsonEncode({ + for (TestResult result in results.values) + result.methodName: result is Failure ? result : success + }); +}