diff --git a/.cirrus.yml b/.cirrus.yml index 415a388ad0..303c3994e7 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -153,7 +153,10 @@ task: # As of October 2019, the Web shards needed more than 6G of RAM. CPU: 2 MEMORY: 8G + GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" + GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] script: + - ./dev/bots/download_goldctl.sh - dart --enable-asserts ./dev/bots/test.dart - name: web_tests-1-linux @@ -162,7 +165,10 @@ task: # As of October 2019, the Web shards needed more than 6G of RAM. CPU: 2 MEMORY: 8G + GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" + GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] script: + - ./dev/bots/download_goldctl.sh - dart --enable-asserts ./dev/bots/test.dart - name: web_tests-2-linux @@ -171,7 +177,10 @@ task: # As of October 2019, the Web shards needed more than 6G of RAM. CPU: 2 MEMORY: 8G + GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" + GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] script: + - ./dev/bots/download_goldctl.sh - dart --enable-asserts ./dev/bots/test.dart - name: web_tests-3-linux @@ -180,7 +189,10 @@ task: # As of October 2019, the Web shards needed more than 6G of RAM. CPU: 2 MEMORY: 8G + GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" + GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] script: + - ./dev/bots/download_goldctl.sh - dart --enable-asserts ./dev/bots/test.dart - name: web_tests-4-linux @@ -189,7 +201,10 @@ task: # As of October 2019, the Web shards needed more than 6G of RAM. CPU: 2 MEMORY: 8G + GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" + GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] script: + - ./dev/bots/download_goldctl.sh - dart --enable-asserts ./dev/bots/test.dart - name: web_tests-5-linux @@ -198,7 +213,10 @@ task: # As of October 2019, the Web shards needed more than 6G of RAM. CPU: 2 MEMORY: 8G + GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" + GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] script: + - ./dev/bots/download_goldctl.sh - dart --enable-asserts ./dev/bots/test.dart - name: web_tests-6-linux @@ -207,7 +225,10 @@ task: # As of October 2019, the Web shards needed more than 6G of RAM. CPU: 2 MEMORY: 8G + GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" + GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] script: + - ./dev/bots/download_goldctl.sh - dart --enable-asserts ./dev/bots/test.dart - name: web_tests-7_last-linux # last Web shard must end with _last @@ -216,7 +237,10 @@ task: # As of October 2019, the Web shards needed more than 6G of RAM. CPU: 2 MEMORY: 8G + GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" + GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] script: + - ./dev/bots/download_goldctl.sh - dart --enable-asserts ./dev/bots/test.dart - name: build_tests-linux diff --git a/dev/bots/test.dart b/dev/bots/test.dart index cf9af11156..bd7de38321 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -68,7 +68,6 @@ const List kWebTestFileBlacklist = [ 'test/widgets/selectable_text_test.dart', 'test/widgets/color_filter_test.dart', 'test/widgets/editable_text_cursor_test.dart', - 'test/widgets/shadow_test.dart', 'test/widgets/raw_keyboard_listener_test.dart', 'test/widgets/editable_text_test.dart', 'test/widgets/widget_inspector_test.dart', diff --git a/packages/flutter/test/widgets/shadow_test.dart b/packages/flutter/test/widgets/shadow_test.dart index b74047346a..e7260e362c 100644 --- a/packages/flutter/test/widgets/shadow_test.dart +++ b/packages/flutter/test/widgets/shadow_test.dart @@ -33,7 +33,7 @@ void main() { matchesGoldenFile('shadow.BoxDecoration.enabled.png'), ); debugDisableShadows = true; - }, skip: isBrowser); + }); testWidgets('Shadows on ShapeDecoration', (WidgetTester tester) async { debugDisableShadows = false; @@ -93,7 +93,7 @@ void main() { matchesGoldenFile('shadow.PhysicalModel.enabled.png'), ); debugDisableShadows = true; - }, skip: isBrowser); + }); testWidgets('Shadows with PhysicalShape', (WidgetTester tester) async { debugDisableShadows = false; diff --git a/packages/flutter_goldens_client/lib/skia_client.dart b/packages/flutter_goldens_client/lib/skia_client.dart index b89ddd1b5a..50a832e575 100644 --- a/packages/flutter_goldens_client/lib/skia_client.dart +++ b/packages/flutter_goldens_client/lib/skia_client.dart @@ -19,6 +19,7 @@ import 'package:process/process.dart'; const String _kFlutterRootKey = 'FLUTTER_ROOT'; const String _kGoldctlKey = 'GOLDCTL'; const String _kServiceAccountKey = 'GOLD_SERVICE_ACCOUNT'; +const String _kTestBrowserKey = 'FLUTTER_TEST_BROWSER'; /// A client for uploading image tests and making baseline requests to the /// Flutter Gold Dashboard. @@ -408,14 +409,16 @@ class SkiaGoldClient { /// Returns a JSON String with keys value pairs used to uniquely identify the /// configuration that generated the given golden file. /// - /// Currently, the only key value pair being tracked is the platform the image - /// was rendered on. + /// Currently, the only key value pairs being tracked is the platform the + /// image was rendered on, and for web tests, the browser the image was + /// rendered on. String _getKeysJSON() { - return json.encode( - { - 'Platform' : platform.operatingSystem, - } - ); + final Map keys = { + 'Platform' : platform.operatingSystem, + }; + if (platform.environment[_kTestBrowserKey] != null) + keys['Browser'] = platform.environment[_kTestBrowserKey]; + return json.encode(keys); } /// Removes the file extension from the [fileName] to represent the test name @@ -455,7 +458,7 @@ class SkiaGoldDigest { return SkiaGoldDigest( imageHash: json['digest'] as String, paramSet: Map.from(json['paramset'] as Map ?? - {'Platform': 'none'}), + >{'Platform': []}), testName: json['test'] as String, status: json['status'] as String, ); @@ -477,6 +480,8 @@ class SkiaGoldDigest { bool isValid(Platform platform, String name, String expectation) { return imageHash == expectation && (paramSet['Platform'] as List).contains(platform.operatingSystem) + && (platform.environment[_kTestBrowserKey] == null + || paramSet['Browser'] == platform.environment[_kTestBrowserKey]) && testName == name && status == 'positive'; } diff --git a/packages/flutter_test/lib/src/_goldens_io.dart b/packages/flutter_test/lib/src/_goldens_io.dart index 06ce06ca7a..6cb88df459 100644 --- a/packages/flutter_test/lib/src/_goldens_io.dart +++ b/packages/flutter_test/lib/src/_goldens_io.dart @@ -6,7 +6,9 @@ import 'dart:async'; import 'dart:io'; import 'dart:math' as math; import 'dart:typed_data'; +import 'dart:ui'; +import 'package:flutter/widgets.dart' show Element; import 'package:image/image.dart'; import 'package:path/path.dart' as path; // ignore: deprecated_member_use @@ -240,3 +242,16 @@ ComparisonResult compareLists(List test, List master) { } return ComparisonResult(passed: true); } + +/// An unsupported [WebGoldenComparator] that exists for API compatibility. +class DefaultWebGoldenComparator extends WebGoldenComparator { + @override + Future compare(Element element, Size size, Uri golden) { + throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.'); + } + + @override + Future update(Uri golden, Element element, Size size) { + throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.'); + } +} diff --git a/packages/flutter_test/lib/src/_goldens_web.dart b/packages/flutter_test/lib/src/_goldens_web.dart index 86c75e0bbf..055a8ff19e 100644 --- a/packages/flutter_test/lib/src/_goldens_web.dart +++ b/packages/flutter_test/lib/src/_goldens_web.dart @@ -2,11 +2,19 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. - import 'dart:typed_data'; +import 'dart:convert'; +import 'dart:html' as html; +import 'dart:typed_data'; +import 'dart:ui'; - import 'goldens.dart'; +import 'package:flutter/rendering.dart'; +import 'package:flutter/widgets.dart'; +// ignore: deprecated_member_use +import 'package:test_api/test_api.dart' as test_package show TestFailure; - /// An unsupported [GoldenFileComparator] that exists for API compatibility. +import 'goldens.dart'; + +/// An unsupported [GoldenFileComparator] that exists for API compatibility. class LocalFileComparator extends GoldenFileComparator { @override Future compare(Uint8List imageBytes, Uri golden) { @@ -19,10 +27,63 @@ class LocalFileComparator extends GoldenFileComparator { } } - /// Returns whether [test] and [master] are pixel by pixel identical. +/// Returns whether [test] and [master] are pixel by pixel identical. /// /// This method is not supported on the web and throws an [UnsupportedError] /// when called. ComparisonResult compareLists(List test, List master) { throw UnsupportedError('Golden testing is not supported on the web.'); } + +/// The default [WebGoldenComparator] implementation for `flutter test`. +/// +/// This comparator will send a request to the test server for golden comparison +/// which will then defer the comparison to [goldenFileComparator]. +/// +/// See also: +/// +/// * [matchesGoldenFile], the function from [flutter_test] that invokes the +/// comparator. +class DefaultWebGoldenComparator extends WebGoldenComparator { + /// Creates a new [DefaultWebGoldenComparator] for the specified [testFile]. + /// + /// Golden file keys will be interpreted as file paths relative to the + /// directory in which [testFile] resides. + /// + /// The [testFile] URL must represent a file. + DefaultWebGoldenComparator(this.testUri); + + /// The test file currently being executed. + /// + /// Golden file keys will be interpreted as file paths relative to the + /// directory in which this file resides. + Uri testUri; + + @override + Future compare(Element element, Size size, Uri golden) async { + final String key = golden.toString(); + + final html.HttpRequest request = await html.HttpRequest.request( + 'flutter_goldens', + method: 'POST', + sendData: json.encode({ + 'testUri': testUri.toString(), + 'key': key.toString(), + 'width': size.width.round(), + 'height': size.height.round(), + }), + ); + final String response = request.response as String; + if (response == 'true') { + return true; + } else { + throw test_package.TestFailure(response); + } + } + + @override + Future update(Uri golden, Element element, Size size) async { + // Update is handled on the server side, just use the same logic here + await compare(element, size, golden); + } +} diff --git a/packages/flutter_test/lib/src/_matchers_io.dart b/packages/flutter_test/lib/src/_matchers_io.dart new file mode 100644 index 0000000000..9af9c7eacd --- /dev/null +++ b/packages/flutter_test/lib/src/_matchers_io.dart @@ -0,0 +1,95 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:typed_data'; +import 'dart:ui' as ui; + +import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; +import 'package:flutter/services.dart'; +import 'package:test_api/src/frontend/async_matcher.dart'; // ignore: implementation_imports +// ignore: deprecated_member_use +import 'package:test_api/test_api.dart' hide TypeMatcher, isInstanceOf; + +import 'binding.dart'; +import 'finders.dart'; +import 'goldens.dart'; + +/// Render the closest [RepaintBoundary] of the [element] into an image. +/// +/// See also: +/// +/// * [OffsetLayer.toImage] which is the actual method being called. +Future captureImage(Element element) { + RenderObject renderObject = element.renderObject; + while (!renderObject.isRepaintBoundary) { + renderObject = renderObject.parent as RenderObject; + assert(renderObject != null); + } + assert(!renderObject.debugNeedsPaint); + final OffsetLayer layer = renderObject.debugLayer as OffsetLayer; + return layer.toImage(renderObject.paintBounds); +} + +/// The matcher created by [matchesGoldenFile]. This class is enabled when the +/// test is running on a VM using conditional import. +class MatchesGoldenFile extends AsyncMatcher { + /// Creates an instance of [MatchesGoldenFile]. Called by [matchesGoldenFile]. + const MatchesGoldenFile(this.key, this.version); + + /// Creates an instance of [MatchesGoldenFile]. Called by [matchesGoldenFile]. + MatchesGoldenFile.forStringPath(String path, this.version) : key = Uri.parse(path); + + /// The [key] to the golden image. + final Uri key; + + /// The [version] of the golden image. + final int version; + + @override + Future matchAsync(dynamic item) async { + Future imageFuture; + if (item is Future) { + imageFuture = item; + } else if (item is ui.Image) { + imageFuture = Future.value(item); + } else { + final Finder finder = item as Finder; + final Iterable elements = finder.evaluate(); + if (elements.isEmpty) { + return 'could not be rendered because no widget was found'; + } else if (elements.length > 1) { + return 'matched too many widgets'; + } + imageFuture = captureImage(elements.single); + } + + final Uri testNameUri = goldenFileComparator.getTestUri(key, version); + + final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized() as TestWidgetsFlutterBinding; + return binding.runAsync(() async { + final ui.Image image = await imageFuture; + final ByteData bytes = await image.toByteData(format: ui.ImageByteFormat.png); + if (bytes == null) + return 'could not encode screenshot.'; + if (autoUpdateGoldenFiles) { + await goldenFileComparator.update(testNameUri, bytes.buffer.asUint8List()); + return null; + } + try { + final bool success = await goldenFileComparator.compare(bytes.buffer.asUint8List(), testNameUri); + return success ? null : 'does not match'; + } on TestFailure catch (ex) { + return ex.message; + } + }, additionalTime: const Duration(minutes: 1)); + } + + @override + Description describe(Description description) { + final Uri testNameUri = goldenFileComparator.getTestUri(key, version); + return description.add('one widget whose rasterized image matches golden image "$testNameUri"'); + } +} diff --git a/packages/flutter_test/lib/src/_matchers_web.dart b/packages/flutter_test/lib/src/_matchers_web.dart new file mode 100644 index 0000000000..31ec185882 --- /dev/null +++ b/packages/flutter_test/lib/src/_matchers_web.dart @@ -0,0 +1,105 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:ui' as ui; + +import 'package:flutter/rendering.dart'; +import 'package:flutter/widgets.dart'; +import 'package:test_api/src/frontend/async_matcher.dart'; // ignore: implementation_imports +// ignore: deprecated_member_use +import 'package:test_api/test_api.dart' hide TypeMatcher, isInstanceOf; + +import 'binding.dart'; +import 'finders.dart'; +import 'goldens.dart'; + +/// An unsupported method that exists for API compatibility. +Future captureImage(Element element) { + throw UnsupportedError('captureImage is not supported on the web.'); +} + +/// The matcher created by [matchesGoldenFile]. This class is enabled when the +/// test is running in a web browser using conditional import. +class MatchesGoldenFile extends AsyncMatcher { + /// Creates an instance of [MatchesGoldenFile]. Called by [matchesGoldenFile]. + const MatchesGoldenFile(this.key, this.version); + + /// Creates an instance of [MatchesGoldenFile]. Called by [matchesGoldenFile]. + MatchesGoldenFile.forStringPath(String path, this.version) : key = Uri.parse(path); + + /// The [key] to the golden image. + final Uri key; + + /// The [version] of the golden image. + final int version; + + @override + Future matchAsync(dynamic item) async { + if (item is! Finder) { + return 'web goldens only supports matching finders.'; + } + final Finder finder = item as Finder; + final Iterable elements = finder.evaluate(); + if (elements.isEmpty) { + return 'could not be rendered because no widget was found'; + } else if (elements.length > 1) { + return 'matched too many widgets'; + } + final Element element = elements.single; + final RenderObject renderObject = _findRepaintBoundary(element); + final Size size = renderObject.paintBounds.size; + final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized() as TestWidgetsFlutterBinding; + final Element e = binding.renderViewElement; + + // Unlike `flutter_tester`, we don't have the ability to render an element + // to an image directly. Instead, we will use `window.render()` to render + // only the element being requested, and send a request to the test server + // requesting it to take a screenshot through the browser's debug interface. + _renderElement(binding.window, renderObject); + final String result = await binding.runAsync(() async { + if (autoUpdateGoldenFiles) { + await webGoldenComparator.update(key, element, size); + return null; + } + try { + final bool success = await webGoldenComparator.compare(element, size, key); + return success ? null : 'does not match'; + } on TestFailure catch (ex) { + return ex.message; + } + }, additionalTime: const Duration(seconds: 11)); + _renderElement(binding.window, _findRepaintBoundary(e)); + return result; + } + + @override + Description describe(Description description) { + final Uri testNameUri = webGoldenComparator.getTestUri(key, version); + return description.add('one widget whose rasterized image matches golden image "$testNameUri"'); + } +} + +RenderObject _findRepaintBoundary(Element element) { + RenderObject renderObject = element.renderObject; + while (!renderObject.isRepaintBoundary) { + renderObject = renderObject.parent as RenderObject; + assert(renderObject != null); + } + return renderObject; +} + +void _renderElement(ui.Window window, RenderObject renderObject) { + final Layer layer = renderObject.debugLayer; + final ui.SceneBuilder sceneBuilder = ui.SceneBuilder(); + if (layer is OffsetLayer) { + sceneBuilder.pushOffset(-layer.offset.dx, -layer.offset.dy); + } + // ignore: invalid_use_of_visible_for_testing_member, invalid_use_of_protected_member + layer.updateSubtreeNeedsAddToScene(); + // ignore: invalid_use_of_protected_member + layer.addToScene(sceneBuilder); + sceneBuilder.pop(); + window.render(sceneBuilder.build()); +} diff --git a/packages/flutter_test/lib/src/goldens.dart b/packages/flutter_test/lib/src/goldens.dart index f62bbb5e19..d390d7dc1e 100644 --- a/packages/flutter_test/lib/src/goldens.dart +++ b/packages/flutter_test/lib/src/goldens.dart @@ -6,6 +6,7 @@ import 'dart:async'; import 'dart:typed_data'; import 'package:flutter/foundation.dart'; +import 'package:flutter/widgets.dart'; import 'package:path/path.dart' as path; import '_goldens_io.dart' if (dart.library.html) '_goldens_web.dart' as _goldens; @@ -134,6 +135,115 @@ set goldenFileComparator(GoldenFileComparator value) { _goldenFileComparator = value; } +/// Compares image pixels against a golden image file. +/// +/// Instances of this comparator will be used as the backend for +/// [matchesGoldenFile] when tests are running on Flutter Web, and will usually +/// implemented by deferring the screenshot taking and image comparison to a +/// test server. +/// +/// Instances of this comparator will be invoked by the test framework in the +/// [TestWidgetsFlutterBinding.runAsync] zone and are thus not subject to the +/// fake async constraints that are normally imposed on widget tests (i.e. the +/// need or the ability to call [WidgetTester.pump] to advance the microtask +/// queue). Prior to the invocation, the test framework will render only the +/// [Element] to be compared on the screen. +/// +/// See also: +/// +/// * [GoldenFileComparator] for the comparator to be used when the test is +/// not running in a web browser. +/// * [DefaultWebGoldenComparator] for the default [WebGoldenComparator] +/// implementation for `flutter test`. +/// * [matchesGoldenFile], the function from [flutter_test] that invokes the +/// comparator. +abstract class WebGoldenComparator { + /// Compares the rendered pixels of [element] of size [size] that is being + /// rendered on the top left of the screen against the golden file identified + /// by [golden]. + /// + /// The returned future completes with a boolean value that indicates whether + /// the pixels rendered on screen match the golden file's pixels. + /// + /// In the case of comparison mismatch, the comparator may choose to throw a + /// [TestFailure] if it wants to control the failure message, often in the + /// form of a [ComparisonResult] that provides detailed information about the + /// mismatch. + /// + /// The method by which [golden] is located and by which its bytes are loaded + /// is left up to the implementation class. For instance, some implementations + /// may load files from the local file system, whereas others may load files + /// over the network or from a remote repository. + Future compare(Element element, Size size, Uri golden); + + /// Updates the golden file identified by [golden] with rendered pixels of + /// [element]. + /// + /// This will be invoked in lieu of [compare] when [autoUpdateGoldenFiles] + /// is `true` (which gets set automatically by the test framework when the + /// user runs `flutter test --update-goldens --platform=chrome`). + /// + /// The method by which [golden] is located and by which its bytes are written + /// is left up to the implementation class. + Future update(Uri golden, Element element, Size size); + + /// Returns a new golden file [Uri] to incorporate any [version] number with + /// the [key]. + /// + /// The [version] is an optional int that can be used to differentiate + /// historical golden files. + /// + /// Version numbers are used in golden file tests for package:flutter. You can + /// learn more about these tests [here](https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter). + Uri getTestUri(Uri key, int version) { + if (version == null) + return key; + final String keyString = key.toString(); + final String extension = path.extension(keyString); + return Uri.parse( + keyString + .split(extension) + .join() + '.' + version.toString() + extension + ); + } +} + +/// Compares pixels against those of a golden image file. +/// +/// This comparator is used as the backend for [matchesGoldenFile] when tests +/// are running in a web browser. +/// +/// When using `flutter test --platform=chrome`, a comparator implemented by +/// [DefaultWebGoldenComparator] is used if no other comparator is specified. It +/// will send a request to the test server, which uses [goldenFileComparator] +/// for golden file compatison. +/// +/// When using `flutter test --update-goldens`, the [DefaultWebGoldenComparator] +/// updates the files on disk to match the rendering. +/// +/// When using `flutter run`, the default comparator +/// ([_TrivialWebGoldenComparator]) is used. It prints a message to the console +/// but otherwise does nothing. This allows tests to be developed visually on a +/// web browser. +/// +/// Callers may choose to override the default comparator by setting this to a +/// custom comparator during test set-up (or using directory-level test +/// configuration). For example, some projects may wish to install a comparator +/// with tolerance levels for allowable differences. +/// +/// See also: +/// +/// * [flutter_test] for more information about how to configure tests at the +/// directory-level. +/// * [goldenFileComparator], the comparator used when tests are not running on +/// a web browser. +WebGoldenComparator get webGoldenComparator => _webGoldenComparator; +WebGoldenComparator _webGoldenComparator = const _TrivialWebGoldenComparator._(); +set webGoldenComparator(WebGoldenComparator value) { + assert(value != null); + _webGoldenComparator = value; +} + /// Whether golden files should be automatically updated during tests rather /// than compared to the image bytes recorded by the tests. /// @@ -185,6 +295,26 @@ class TrivialComparator implements GoldenFileComparator { } } +class _TrivialWebGoldenComparator implements WebGoldenComparator { + const _TrivialWebGoldenComparator._(); + + @override + Future compare(Element element, Size size, Uri golden) { + debugPrint('Golden comparison requested for "$golden"; skipping...'); + return Future.value(true); + } + + @override + Future update(Uri golden, Element element, Size size) { + throw StateError('webGoldenComparator has not been initialized'); + } + + @override + Uri getTestUri(Uri key, int version) { + return key; + } +} + /// The result of a pixel comparison test. /// /// The [ComparisonResult] will always indicate if a test has [passed]. The diff --git a/packages/flutter_test/lib/src/matchers.dart b/packages/flutter_test/lib/src/matchers.dart index 4ef3a13b01..866e3d08a0 100644 --- a/packages/flutter_test/lib/src/matchers.dart +++ b/packages/flutter_test/lib/src/matchers.dart @@ -20,6 +20,7 @@ import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter/services.dart'; +import '_matchers_io.dart' if (dart.library.html) '_matchers_web.dart' show MatchesGoldenFile, captureImage; import 'accessibility.dart'; import 'binding.dart'; import 'finders.dart'; @@ -366,9 +367,9 @@ Matcher coversSameAreaAs(Path expectedPath, { @required Rect areaToCompare, int /// may swap out the backend for this matcher. AsyncMatcher matchesGoldenFile(dynamic key, {int version}) { if (key is Uri) { - return _MatchesGoldenFile(key, version); + return MatchesGoldenFile(key, version); } else if (key is String) { - return _MatchesGoldenFile.forStringPath(key, version); + return MatchesGoldenFile.forStringPath(key, version); } throw ArgumentError('Unexpected type for golden file: ${key.runtimeType}'); } @@ -1636,17 +1637,6 @@ class _ColorMatcher extends Matcher { Description describe(Description description) => description.add('matches color $targetColor'); } -Future _captureImage(Element element) { - RenderObject renderObject = element.renderObject; - while (!renderObject.isRepaintBoundary) { - renderObject = renderObject.parent as RenderObject; - assert(renderObject != null); - } - assert(!renderObject.debugNeedsPaint); - final OffsetLayer layer = renderObject.debugLayer as OffsetLayer; - return layer.toImage(renderObject.paintBounds); -} - int _countDifferentPixels(Uint8List imageA, Uint8List imageB) { assert(imageA.length == imageB.length); int delta = 0; @@ -1681,7 +1671,7 @@ class _MatchesReferenceImage extends AsyncMatcher { } else if (elements.length > 1) { return 'matched too many widgets'; } - imageFuture = _captureImage(elements.single); + imageFuture = captureImage(elements.single); } final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized() as TestWidgetsFlutterBinding; @@ -1712,60 +1702,6 @@ class _MatchesReferenceImage extends AsyncMatcher { } } -class _MatchesGoldenFile extends AsyncMatcher { - const _MatchesGoldenFile(this.key, this.version); - - _MatchesGoldenFile.forStringPath(String path, this.version) : key = Uri.parse(path); - - final Uri key; - final int version; - - @override - Future matchAsync(dynamic item) async { - Future imageFuture; - if (item is Future) { - imageFuture = item; - } else if (item is ui.Image) { - imageFuture = Future.value(item); - } else { - final Finder finder = item as Finder; - final Iterable elements = finder.evaluate(); - if (elements.isEmpty) { - return 'could not be rendered because no widget was found'; - } else if (elements.length > 1) { - return 'matched too many widgets'; - } - imageFuture = _captureImage(elements.single); - } - - final Uri testNameUri = goldenFileComparator.getTestUri(key, version); - - final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized() as TestWidgetsFlutterBinding; - return binding.runAsync(() async { - final ui.Image image = await imageFuture; - final ByteData bytes = await image.toByteData(format: ui.ImageByteFormat.png); - if (bytes == null) - return 'could not encode screenshot.'; - if (autoUpdateGoldenFiles) { - await goldenFileComparator.update(testNameUri, bytes.buffer.asUint8List()); - return null; - } - try { - final bool success = await goldenFileComparator.compare(bytes.buffer.asUint8List(), testNameUri); - return success ? null : 'does not match'; - } on TestFailure catch (ex) { - return ex.message; - } - }, additionalTime: const Duration(minutes: 1)); - } - - @override - Description describe(Description description) { - final Uri testNameUri = goldenFileComparator.getTestUri(key, version); - return description.add('one widget whose rasterized image matches golden image "$testNameUri"'); - } -} - class _MatchesSemanticsData extends Matcher { _MatchesSemanticsData({ this.label, diff --git a/packages/flutter_tools/lib/src/build_runner/build_script.dart b/packages/flutter_tools/lib/src/build_runner/build_script.dart index 89cf257d7f..757206c14d 100644 --- a/packages/flutter_tools/lib/src/build_runner/build_script.dart +++ b/packages/flutter_tools/lib/src/build_runner/build_script.dart @@ -255,6 +255,7 @@ class FlutterWebTestBootstrapBuilder implements Builder { final String assetPath = id.pathSegments.first == 'lib' ? path.url.join('packages', id.package, id.path) : id.path; + final Uri testUrl = path.toUri(path.absolute(assetPath)); final Metadata metadata = parseMetadata( assetPath, contents, Runtime.builtIn.map((Runtime runtime) => runtime.name).toSet()); @@ -265,6 +266,7 @@ import 'dart:html'; import 'dart:js'; import 'package:stream_channel/stream_channel.dart'; +import 'package:flutter_test/flutter_test.dart'; import 'package:test_api/src/backend/stack_trace_formatter.dart'; // ignore: implementation_imports import 'package:test_api/src/util/stack_trace_mapper.dart'; // ignore: implementation_imports import 'package:test_api/src/remote_listener.dart'; // ignore: implementation_imports @@ -279,6 +281,7 @@ Future main() async { // this stuff in. ui.debugEmulateFlutterTesterEnvironment = true; await ui.webOnlyInitializePlatform(); + webGoldenComparator = DefaultWebGoldenComparator(Uri.parse('$testUrl')); // TODO(flutterweb): remove need for dynamic cast. (ui.window as dynamic).debugOverrideDevicePixelRatio(3.0); (ui.window as dynamic).webOnlyDebugPhysicalSizeOverride = const ui.Size(2400, 1800); diff --git a/packages/flutter_tools/lib/src/test/flutter_platform.dart b/packages/flutter_tools/lib/src/test/flutter_platform.dart index 9b7c903464..3952fcc08e 100644 --- a/packages/flutter_tools/lib/src/test/flutter_platform.dart +++ b/packages/flutter_tools/lib/src/test/flutter_platform.dart @@ -29,6 +29,7 @@ import '../globals.dart'; import '../project.dart'; import '../vmservice.dart'; import 'test_compiler.dart'; +import 'test_config.dart'; import 'watcher.dart'; /// The timeout we give the test process to connect to the test harness @@ -55,14 +56,6 @@ const Duration _kTestProcessTimeout = Duration(minutes: 5); /// hold that against the test. const String _kStartTimeoutTimerMessage = 'sky_shell test process has entered main method'; -/// The name of the test configuration file that will be discovered by the -/// test harness if it exists in the project directory hierarchy. -const String _kTestConfigFileName = 'flutter_test_config.dart'; - -/// The name of the file that signals the root of the project and that will -/// cause the test harness to stop scanning for configuration files. -const String _kProjectRootSentinel = 'pubspec.yaml'; - /// The address at which our WebSocket server resides and at which the sky_shell /// processes will host the Observatory server. final Map _kHosts = { @@ -743,25 +736,9 @@ class FlutterPlatform extends PlatformPlugin { Uri testUrl, }) { assert(testUrl.scheme == 'file'); - File testConfigFile; - Directory directory = fs.file(testUrl).parent; - while (directory.path != directory.parent.path) { - final File configFile = directory.childFile(_kTestConfigFileName); - if (configFile.existsSync()) { - printTrace('Discovered $_kTestConfigFileName in ${directory.path}'); - testConfigFile = configFile; - break; - } - if (directory.childFile(_kProjectRootSentinel).existsSync()) { - printTrace('Stopping scan for $_kTestConfigFileName; ' - 'found project root at ${directory.path}'); - break; - } - directory = directory.parent; - } return generateTestBootstrap( testUrl: testUrl, - testConfigFile: testConfigFile, + testConfigFile: findTestConfigFile(fs.file(testUrl)), host: host, updateGoldens: updateGoldens, ); diff --git a/packages/flutter_tools/lib/src/test/flutter_web_platform.dart b/packages/flutter_tools/lib/src/test/flutter_web_platform.dart index 82eef852fc..72d6cafbad 100644 --- a/packages/flutter_tools/lib/src/test/flutter_web_platform.dart +++ b/packages/flutter_tools/lib/src/test/flutter_web_platform.dart @@ -5,6 +5,7 @@ // ignore_for_file: implementation_imports import 'dart:async'; +import 'dart:typed_data'; import 'package:async/async.dart'; import 'package:http_multi_server/http_multi_server.dart'; @@ -27,18 +28,30 @@ import 'package:test_core/src/runner/plugin/platform_helpers.dart'; import 'package:test_core/src/runner/runner_suite.dart'; import 'package:test_core/src/runner/suite.dart'; import 'package:web_socket_channel/web_socket_channel.dart'; +import 'package:webkit_inspection_protocol/webkit_inspection_protocol.dart' hide StackTrace; import '../artifacts.dart'; import '../base/common.dart'; import '../base/file_system.dart'; +import '../base/io.dart'; +import '../base/process_manager.dart'; +import '../build_info.dart'; import '../cache.dart'; import '../convert.dart'; import '../dart/package_map.dart'; import '../globals.dart'; +import '../project.dart'; import '../web/chrome.dart'; +import 'test_compiler.dart'; +import 'test_config.dart'; + class FlutterWebPlatform extends PlatformPlugin { - FlutterWebPlatform._(this._server, this._config, this._root) { + FlutterWebPlatform._(this._server, this._config, this._root, { + FlutterProject flutterProject, + String shellPath, + this.updateGoldens, + }) { // Look up the location of the testing resources. final Map packageMap = PackageMap(fs.path.join( Cache.flutterRoot, @@ -58,17 +71,30 @@ class FlutterWebPlatform extends PlatformPlugin { .add(createStaticHandler(_config.suiteDefaults.precompiledPath, serveFilesOutsidePath: true)) .add(_handleStaticArtifact) + .add(_goldenFileHandler) .add(_wrapperHandler); _server.mount(cascade.handler); + + _testGoldenComparator = TestGoldenComparator( + shellPath, + () => TestCompiler(BuildMode.debug, false, flutterProject), + ); } - static Future start(String root) async { + static Future start(String root, { + FlutterProject flutterProject, + String shellPath, + bool updateGoldens = false, + }) async { final shelf_io.IOServer server = shelf_io.IOServer(await HttpMultiServer.loopback(0)); return FlutterWebPlatform._( server, Configuration.current, root, + flutterProject: flutterProject, + shellPath: shellPath, + updateGoldens: updateGoldens, ); } @@ -167,6 +193,59 @@ class FlutterWebPlatform extends PlatformPlugin { } } + final bool updateGoldens; + TestGoldenComparator _testGoldenComparator; + + Future _goldenFileHandler(shelf.Request request) async { + if (request.url.path.contains('flutter_goldens')) { + final Map body = json.decode(await request.readAsString()) as Map; + final Uri goldenKey = Uri.parse(body['key'] as String); + final Uri testUri = Uri.parse(body['testUri'] as String); + final num width = body['width'] as num; + final num height = body['height'] as num; + Uint8List bytes; + + try { + final Runtime browser = Runtime.chrome; + final BrowserManager browserManager = await _browserManagerFor(browser); + final ChromeTab chromeTab = await browserManager._browser.chromeConnection.getTab((ChromeTab tab) { + return tab.url.contains(browserManager._browser.url); + }); + final WipConnection connection = await chromeTab.connect(); + final WipResponse response = await connection.sendCommand('Page.captureScreenshot', { + // Clip the screenshot to include only the element. + // Prior to taking a screenshot, we are calling `window.render()` in + // `_matchers_web.dart` to only render the element on screen. That + // will make sure that the element will always be displayed on the + // origin of the screen. + 'clip': { + 'x': 0.0, + 'y': 0.0, + 'width': width.toDouble(), + 'height': height.toDouble(), + 'scale': 1.0, + } + }); + bytes = base64.decode(response.result['data'] as String); + } on WipError catch (ex) { + printError('Caught WIPError: $ex'); + return shelf.Response.ok('WIP error: $ex'); + } on FormatException catch (ex) { + printError('Caught FormatException: $ex'); + return shelf.Response.ok('Caught exception: $ex'); + } + + if (bytes == null) { + return shelf.Response.ok('Unknown error, bytes is null'); + } + + final String errorMessage = await _testGoldenComparator.compareGoldens(testUri, bytes, goldenKey, updateGoldens); + return shelf.Response.ok(errorMessage ?? 'true'); + } else { + return shelf.Response.notFound('Not Found'); + } + } + final OneOffHandler _webSocketHandler = OneOffHandler(); final PathHandler _jsHandler = PathHandler(); final AsyncMemoizer _closeMemo = AsyncMemoizer(); @@ -296,6 +375,7 @@ class FlutterWebPlatform extends PlatformPlugin { }) .toList(); futures.add(_server.close()); + futures.add(_testGoldenComparator.close()); await Future.wait(futures); }); } @@ -702,3 +782,182 @@ class _BrowserEnvironment implements Environment { @override CancelableOperation displayPause() => _manager._displayPause(); } + +/// Helper class to start golden file comparison in a separate process. +/// +/// Golden file comparator is configured using flutter_test_config.dart and that +/// file can contain arbitrary Dart code that depends on dart:ui. Thus it has to +/// be executed in a `flutter_tester` environment. This helper class generates a +/// Dart file configured with flutter_test_config.dart to perform the comparison +/// of golden files. +class TestGoldenComparator { + /// Creates a [TestGoldenComparator] instance. + TestGoldenComparator(this.shellPath, this.compilerFactory) + : tempDir = fs.systemTempDirectory.createTempSync('flutter_web_platform.'); + + final String shellPath; + final Directory tempDir; + final TestCompiler Function() compilerFactory; + + TestCompiler _compiler; + TestGoldenComparatorProcess _previousComparator; + Uri _previousTestUri; + + Future close() async { + tempDir.deleteSync(recursive: true); + await _compiler?.dispose(); + await _previousComparator?.close(); + } + + /// Start golden comparator in a separate process. Start one file per test file + /// to reduce the overhead of starting `flutter_tester`. + Future _processForTestFile(Uri testUri) async { + if (testUri == _previousTestUri) { + return _previousComparator; + } + + final String bootstrap = TestGoldenComparatorProcess.generateBootstrap(testUri); + final Process process = await _startProcess(bootstrap); + unawaited(_previousComparator?.close()); + _previousComparator = TestGoldenComparatorProcess(process); + _previousTestUri = testUri; + + return _previousComparator; + } + + Future _startProcess(String testBootstrap) async { + // Prepare the Dart file that will talk to us and start the test. + final File listenerFile = (await tempDir.createTemp('listener')).childFile('listener.dart'); + await listenerFile.writeAsString(testBootstrap); + + // Lazily create the compiler + _compiler = _compiler ?? compilerFactory(); + final String output = await _compiler.compile(listenerFile.path); + final List command = [ + shellPath, + '--disable-observatory', + '--non-interactive', + '--packages=${PackageMap.globalPackagesPath}', + output, + ]; + + final Map environment = { + // Chrome is the only supported browser currently. + 'FLUTTER_TEST_BROWSER': 'chrome', + }; + return processManager.start(command, environment: environment); + } + + Future compareGoldens(Uri testUri, Uint8List bytes, Uri goldenKey, bool updateGoldens) async { + final File imageFile = await (await tempDir.createTemp('image')).childFile('image').writeAsBytes(bytes); + + final TestGoldenComparatorProcess process = await _processForTestFile(testUri); + process.sendCommand(imageFile, goldenKey, updateGoldens); + + final Map result = await process.getResponse().timeout(const Duration(seconds: 10)); + + if (result == null) { + return 'unknown error'; + } else { + return (result['success'] as bool) ? null : ((result['message'] as String) ?? 'does not match'); + } + } +} + +/// Represents a `flutter_tester` process started for golden comparison. Also +/// handles communication with the child process. +class TestGoldenComparatorProcess { + /// Creates a [TestGoldenComparatorProcess] backed by [process]. + TestGoldenComparatorProcess(this.process) { + // Pipe stdout and stderr to printTrace and printError. + // Also parse stdout as a stream of JSON objects. + streamIterator = StreamIterator>( + process.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .where((String line) { + printTrace('<<< $line'); + return line.isNotEmpty && line[0] == '{'; + }) + .map(jsonDecode) + .cast>()); + + process.stderr + .transform(utf8.decoder) + .transform(const LineSplitter()) + .forEach((String line) { + printError('<<< $line'); + }); + } + + final Process process; + StreamIterator> streamIterator; + + Future close() async { + await process.stdin.close(); + process.kill(); + } + + void sendCommand(File imageFile, Uri goldenKey, bool updateGoldens) { + final Object command = jsonEncode({ + 'imageFile': imageFile.path, + 'key': goldenKey.toString(), + 'update': updateGoldens, + }); + printTrace('Preparing to send command: $command'); + process.stdin.writeln(command); + } + + Future> getResponse() async { + final bool available = await streamIterator.moveNext(); + assert(available); + return streamIterator.current; + } + + static String generateBootstrap(Uri testUri) { + final File testConfigFile = findTestConfigFile(fs.file(testUri)); + // Generate comparator process for the file. + return ''' +import 'dart:convert'; // ignore: dart_convert_import +import 'dart:io'; // ignore: dart_io_import + +import 'package:flutter_test/flutter_test.dart'; + +${testConfigFile != null ? "import '${Uri.file(testConfigFile.path)}' as test_config;" : ""} + +void main() async { + LocalFileComparator comparator = LocalFileComparator(Uri.parse('$testUri')); + goldenFileComparator = comparator; + + ${testConfigFile != null ? 'test_config.main(() async {' : ''} + final commands = stdin + .transform(utf8.decoder) + .transform(const LineSplitter()) + .map(jsonDecode); + await for (Object command in commands) { + if (command is Map) { + File imageFile = File(command['imageFile']); + Uri goldenKey = Uri.parse(command['key']); + bool update = command['update']; + + final bytes = await File(imageFile.path).readAsBytes(); + if (update) { + await goldenFileComparator.update(goldenKey, bytes); + print(jsonEncode({'success': true})); + } else { + try { + bool success = await goldenFileComparator.compare(bytes, goldenKey); + print(jsonEncode({'success': success})); + } catch (ex) { + print(jsonEncode({'success': false, 'message': '\$ex'})); + } + } + } else { + print('object type is not right'); + } + } + ${testConfigFile != null ? '});' : ''} +} + '''; + } +} diff --git a/packages/flutter_tools/lib/src/test/runner.dart b/packages/flutter_tools/lib/src/test/runner.dart index 42241ae05c..58c4e52a7a 100644 --- a/packages/flutter_tools/lib/src/test/runner.dart +++ b/packages/flutter_tools/lib/src/test/runner.dart @@ -48,6 +48,12 @@ Future runTests( Directory coverageDirectory, bool web = false, }) async { + // Configure package:test to use the Flutter engine for child processes. + final String shellPath = artifacts.getArtifactPath(Artifact.flutterTester); + if (!processManager.canRun(shellPath)) { + throwToolExit('Cannot execute Flutter tester at $shellPath'); + } + // Compute the command-line arguments for package:test. final List testArgs = [ if (!terminal.supportsColor) @@ -86,7 +92,12 @@ Future runTests( hack.registerPlatformPlugin( [Runtime.chrome], () { - return FlutterWebPlatform.start(flutterProject.directory.path); + return FlutterWebPlatform.start( + flutterProject.directory.path, + updateGoldens: updateGoldens, + shellPath: shellPath, + flutterProject: flutterProject, + ); }, ); await test.main(testArgs); @@ -97,12 +108,6 @@ Future runTests( ..add('--') ..addAll(testFiles); - // Configure package:test to use the Flutter engine for child processes. - final String shellPath = artifacts.getArtifactPath(Artifact.flutterTester); - if (!processManager.canRun(shellPath)) { - throwToolExit('Cannot find Flutter shell at $shellPath'); - } - final InternetAddressType serverType = ipv6 ? InternetAddressType.IPv6 : InternetAddressType.IPv4; diff --git a/packages/flutter_tools/lib/src/test/test_config.dart b/packages/flutter_tools/lib/src/test/test_config.dart new file mode 100644 index 0000000000..1d65774a06 --- /dev/null +++ b/packages/flutter_tools/lib/src/test/test_config.dart @@ -0,0 +1,35 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import '../base/file_system.dart'; +import '../globals.dart'; + +/// The name of the test configuration file that will be discovered by the +/// test harness if it exists in the project directory hierarchy. +const String _kTestConfigFileName = 'flutter_test_config.dart'; + +/// The name of the file that signals the root of the project and that will +/// cause the test harness to stop scanning for configuration files. +const String _kProjectRootSentinel = 'pubspec.yaml'; + +/// Find the `flutter_test_config.dart` file for a specific test file. +File findTestConfigFile(File testFile) { + File testConfigFile; + Directory directory = testFile.parent; + while (directory.path != directory.parent.path) { + final File configFile = directory.childFile(_kTestConfigFileName); + if (configFile.existsSync()) { + printTrace('Discovered $_kTestConfigFileName in ${directory.path}'); + testConfigFile = configFile; + break; + } + if (directory.childFile(_kProjectRootSentinel).existsSync()) { + printTrace('Stopping scan for $_kTestConfigFileName; ' + 'found project root at ${directory.path}'); + break; + } + directory = directory.parent; + } + return testConfigFile; +} diff --git a/packages/flutter_tools/lib/src/web/chrome.dart b/packages/flutter_tools/lib/src/web/chrome.dart index 03d5d568dd..f1e926d774 100644 --- a/packages/flutter_tools/lib/src/web/chrome.dart +++ b/packages/flutter_tools/lib/src/web/chrome.dart @@ -137,6 +137,7 @@ class ChromeLauncher { '--no-default-browser-check', '--disable-default-apps', '--disable-translate', + '--window-size=2400,1800', if (headless) ...['--headless', '--disable-gpu', '--no-sandbox'], url, @@ -174,6 +175,7 @@ class ChromeLauncher { return _connect(Chrome._( port, ChromeConnection('localhost', port), + url: url, process: process, remoteDebuggerUri: remoteDebuggerUri, ), skipCheck); @@ -225,10 +227,12 @@ class Chrome { Chrome._( this.debugPort, this.chromeConnection, { + this.url, Process process, this.remoteDebuggerUri, }) : _process = process; + final String url; final int debugPort; final Process _process; final ChromeConnection chromeConnection; diff --git a/packages/flutter_tools/static/index.html b/packages/flutter_tools/static/index.html index 8e5f9721e3..85ebd46c4c 100644 --- a/packages/flutter_tools/static/index.html +++ b/packages/flutter_tools/static/index.html @@ -4,23 +4,31 @@ Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. --> - test Browser Host + Flutter Test Browser Host + - - - - - - - -
- - - - - diff --git a/packages/flutter_tools/test/general.shard/web/golden_comparator_process_test.dart b/packages/flutter_tools/test/general.shard/web/golden_comparator_process_test.dart new file mode 100644 index 0000000000..2cc377b9d0 --- /dev/null +++ b/packages/flutter_tools/test/general.shard/web/golden_comparator_process_test.dart @@ -0,0 +1,113 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; + +import 'package:flutter_tools/src/base/file_system.dart'; +import 'package:flutter_tools/src/test/flutter_web_platform.dart'; + +import '../../src/common.dart'; +import '../../src/mocks.dart'; +import '../../src/testbed.dart'; + +void main() { + final Testbed testbed = Testbed(); + + group('Test that TestGoldenComparatorProcess', () { + File imageFile; + Uri goldenKey; + File imageFile2; + Uri goldenKey2; + MockProcess Function(String) createMockProcess; + + setUpAll(() { + imageFile = fs.file('test_image_file'); + goldenKey = Uri.parse('file://golden_key'); + imageFile2 = fs.file('second_test_image_file'); + goldenKey2 = Uri.parse('file://second_golden_key'); + createMockProcess = (String stdout) => MockProcess( + exitCode: Future.value(0), + stdout: stdoutFromString(stdout), + ); + }); + + test('can pass data', () => testbed.run(() async { + final Map expectedResponse = { + 'success': true, + 'message': 'some message', + }; + + final MockProcess mockProcess = createMockProcess(jsonEncode(expectedResponse) + '\n'); + final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink; + + final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess); + process.sendCommand(imageFile, goldenKey, false); + + final Map response = await process.getResponse(); + final String stringToStdin = stringFromMemoryIOSink(ioSink); + + expect(response, expectedResponse); + expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n'); + })); + + test('can handle multiple requests', () => testbed.run(() async { + final Map expectedResponse1 = { + 'success': true, + 'message': 'some message', + }; + final Map expectedResponse2 = { + 'success': false, + 'message': 'some other message', + }; + + final MockProcess mockProcess = createMockProcess(jsonEncode(expectedResponse1) + '\n' + jsonEncode(expectedResponse2) + '\n'); + final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink; + + final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess); + process.sendCommand(imageFile, goldenKey, false); + + final Map response1 = await process.getResponse(); + + process.sendCommand(imageFile2, goldenKey2, true); + + final Map response2 = await process.getResponse(); + final String stringToStdin = stringFromMemoryIOSink(ioSink); + + expect(response1, expectedResponse1); + expect(response2, expectedResponse2); + expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n{"imageFile":"second_test_image_file","key":"file://second_golden_key/","update":true}\n'); + })); + + test('ignores anything that does not look like JSON', () => testbed.run(() async { + final Map expectedResponse = { + 'success': true, + 'message': 'some message', + }; + + final MockProcess mockProcess = createMockProcess(''' +Some random data including {} curly bracket + {} curly bracket that is not on the beginning of the line +${jsonEncode(expectedResponse)} +{"success": false} +Other JSON data after the initial data +'''); + final MemoryIOSink ioSink = mockProcess.stdin as MemoryIOSink; + + final TestGoldenComparatorProcess process = TestGoldenComparatorProcess(mockProcess); + process.sendCommand(imageFile, goldenKey, false); + + final Map response = await process.getResponse(); + final String stringToStdin = stringFromMemoryIOSink(ioSink); + + expect(response, expectedResponse); + expect(stringToStdin, '{"imageFile":"test_image_file","key":"file://golden_key/","update":false}\n'); + })); + }); +} + +Stream> stdoutFromString(String string) => Stream>.fromIterable(>[ + utf8.encode(string), +]); + +String stringFromMemoryIOSink(MemoryIOSink ioSink) => utf8.decode(ioSink.writes.expand((List l) => l).toList()); diff --git a/packages/flutter_tools/test/general.shard/web/golden_comparator_test.dart b/packages/flutter_tools/test/general.shard/web/golden_comparator_test.dart new file mode 100644 index 0000000000..e2e1c742e1 --- /dev/null +++ b/packages/flutter_tools/test/general.shard/web/golden_comparator_test.dart @@ -0,0 +1,188 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter_tools/src/base/file_system.dart'; +import 'package:flutter_tools/src/test/flutter_web_platform.dart'; +import 'package:flutter_tools/src/test/test_compiler.dart'; +import 'package:mockito/mockito.dart'; +import 'package:process/process.dart'; + +import '../../src/common.dart'; +import '../../src/mocks.dart'; +import '../../src/testbed.dart'; + +void main() { + + group('Test that TestGoldenComparator', () { + Testbed testbed; + Uri goldenKey; + Uri goldenKey2; + Uri testUri; + Uri testUri2; + Uint8List imageBytes; + MockProcessManager mockProcessManager; + MockTestCompiler mockCompiler; + + setUp(() { + goldenKey = Uri.parse('file://golden_key'); + goldenKey2 = Uri.parse('file://second_golden_key'); + testUri = Uri.parse('file://test_uri'); + testUri2 = Uri.parse('file://second_test_uri'); + imageBytes = Uint8List.fromList([1,2,3,4,5]); + mockProcessManager = MockProcessManager(); + mockCompiler = MockTestCompiler(); + when(mockCompiler.compile(any)).thenAnswer((_) => Future.value('compiler_output')); + + testbed = Testbed(overrides: { + ProcessManager: () { + print('in get process manager'); + return mockProcessManager; + } + }); + }); + + test('succeed when golden comparison succeed', () => testbed.run(() async { + final Map expectedResponse = { + 'success': true, + 'message': 'some message', + }; + + when(mockProcessManager.start(any, environment: anyNamed('environment'))) + .thenAnswer((Invocation invocation) async { + return FakeProcess( + exitCode: Future.value(0), + stdout: stdoutFromString(jsonEncode(expectedResponse) + '\n'), + ); + }); + + final TestGoldenComparator comparator = TestGoldenComparator( + 'shell', + () => mockCompiler, + ); + + final String result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false); + expect(result, null); + })); + + test('fail with error message when golden comparison failed', () => testbed.run(() async { + final Map expectedResponse = { + 'success': false, + 'message': 'some message', + }; + + when(mockProcessManager.start(any, environment: anyNamed('environment'))) + .thenAnswer((Invocation invocation) async { + return FakeProcess( + exitCode: Future.value(0), + stdout: stdoutFromString(jsonEncode(expectedResponse) + '\n'), + ); + }); + + final TestGoldenComparator comparator = TestGoldenComparator( + 'shell', + () => mockCompiler, + ); + + final String result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false); + expect(result, 'some message'); + })); + + test('reuse the process for the same test file', () => testbed.run(() async { + final Map expectedResponse1 = { + 'success': false, + 'message': 'some message', + }; + final Map expectedResponse2 = { + 'success': false, + 'message': 'some other message', + }; + + when(mockProcessManager.start(any, environment: anyNamed('environment'))) + .thenAnswer((Invocation invocation) async { + return FakeProcess( + exitCode: Future.value(0), + stdout: stdoutFromString(jsonEncode(expectedResponse1) + '\n' + jsonEncode(expectedResponse2) + '\n'), + ); + }); + + final TestGoldenComparator comparator = TestGoldenComparator( + 'shell', + () => mockCompiler, + ); + + final String result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false); + expect(result1, 'some message'); + final String result2 = await comparator.compareGoldens(testUri, imageBytes, goldenKey2, false); + expect(result2, 'some other message'); + verify(mockProcessManager.start(any, environment: anyNamed('environment'))).called(1); + })); + + test('does not reuse the process for different test file', () => testbed.run(() async { + final Map expectedResponse1 = { + 'success': false, + 'message': 'some message', + }; + final Map expectedResponse2 = { + 'success': false, + 'message': 'some other message', + }; + + when(mockProcessManager.start(any, environment: anyNamed('environment'))) + .thenAnswer((Invocation invocation) async { + return FakeProcess( + exitCode: Future.value(0), + stdout: stdoutFromString(jsonEncode(expectedResponse1) + '\n' + jsonEncode(expectedResponse2) + '\n'), + ); + }); + + final TestGoldenComparator comparator = TestGoldenComparator( + 'shell', + () => mockCompiler, + ); + + final String result1 = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false); + expect(result1, 'some message'); + final String result2 = await comparator.compareGoldens(testUri2, imageBytes, goldenKey2, false); + expect(result2, 'some message'); + verify(mockProcessManager.start(any, environment: anyNamed('environment'))).called(2); + })); + + test('removes all temporary files when closed', () => testbed.run(() async { + final Map expectedResponse = { + 'success': true, + 'message': 'some message', + }; + + when(mockProcessManager.start(any, environment: anyNamed('environment'))) + .thenAnswer((Invocation invocation) async { + return FakeProcess( + exitCode: Future.value(0), + stdout: stdoutFromString(jsonEncode(expectedResponse) + '\n'), + ); + }); + + final TestGoldenComparator comparator = TestGoldenComparator( + 'shell', + () => mockCompiler, + ); + + final String result = await comparator.compareGoldens(testUri, imageBytes, goldenKey, false); + expect(result, null); + + await comparator.close(); + expect(fs.systemTempDirectory.listSync(recursive: true), isEmpty); + })); + }); +} + +Stream> stdoutFromString(String string) => Stream>.fromIterable(>[ + utf8.encode(string), +]); + +class MockProcessManager extends Mock implements ProcessManager {} +class MockTestCompiler extends Mock implements TestCompiler {}