diff --git a/packages/flutter_goldens/lib/flutter_goldens.dart b/packages/flutter_goldens/lib/flutter_goldens.dart index c6b63eecb9..a0ce417613 100644 --- a/packages/flutter_goldens/lib/flutter_goldens.dart +++ b/packages/flutter_goldens/lib/flutter_goldens.dart @@ -2,10 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:async'; -import 'dart:io' as io; -import 'dart:math' as math; -import 'dart:typed_data'; +import 'dart:async' show FutureOr; +import 'dart:io' as io show OSError, SocketException; +import 'dart:math' as math show Random; +import 'dart:typed_data' show Uint8List; import 'package:file/file.dart'; import 'package:file/local.dart'; @@ -449,13 +449,10 @@ class _UnauthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileCompar if (await skiaClient.imgtestCheck(golden.path, goldenFile)) return true; - // We do not have a matching image, so we need to check a few things - // manually. We wait until this point to do this work so request traffic - // low. - skiaClient.getExpectations(); + // We do not have a matching image hash, so we need to check manually. final String testName = skiaClient.cleanTestName(golden.path); - final List? testExpectations = skiaClient.expectations[testName]; - if (testExpectations == null) { + final String? testExpectation = await skiaClient.getExpectationForTest(testName); + if (testExpectation == null) { // This is a new test. print('No expectations provided by Skia Gold for test: $golden. ' 'This may be a new test. If this is an unexpected result, check ' @@ -466,11 +463,26 @@ class _UnauthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileCompar // Contributors without the proper permissions to execute a tryjob can make // a golden file change through Gold's ignore feature instead. + String? pullRequest; + switch(skiaClient.ci) { + case ContinuousIntegrationEnvironment.cirrus: + pullRequest = platform.environment['CIRRUS_PR']!; + break; + case ContinuousIntegrationEnvironment.luci: + final List refs = platform.environment['GOLD_TRYJOB']!.split('/'); + pullRequest = refs[refs.length - 2]; + break; + case ContinuousIntegrationEnvironment.none: + pullRequest = ''; + break; + } + final bool ignoreResult = await skiaClient.testIsIgnoredForPullRequest( - platform.environment['CIRRUS_PR'] ?? '', + pullRequest, golden.path, ); - // If true, this is an intended change. + // If true, this is an intended change and is being handled on the Flutter + // Gold dashboard: https://flutter-gold.skia.org/ignores return ignoreResult; } } @@ -479,8 +491,7 @@ class _UnauthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileCompar /// golden file tests. /// /// Currently, this comparator is used in some Cirrus test shards and Luci -/// environments, as well as when an internet connection is not available for -/// contacting Gold. +/// environments. /// /// See also: /// @@ -608,9 +619,9 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC } goldens ??= SkiaGoldClient(baseDirectory, ci: ContinuousIntegrationEnvironment.none); - try { - await goldens.getExpectations(); + // Check if we can reach Gold. + await goldens.getExpectationForTest(''); } on io.OSError catch (_) { return FlutterSkippingFileComparator( baseDirectory.uri, @@ -634,8 +645,10 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC Future compare(Uint8List imageBytes, Uri golden) async { golden = _addPrefix(golden); final String testName = skiaClient.cleanTestName(golden.path); - final List? testExpectations = skiaClient.expectations[testName]; - if (testExpectations == null) { + late String? testExpectation; + testExpectation = await skiaClient.getExpectationForTest(testName); + + if (testExpectation == null) { // There is no baseline for this test print('No expectations provided by Skia Gold for test: $golden. ' 'This may be a new test. If this is an unexpected result, check ' @@ -647,25 +660,17 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC } ComparisonResult result; - final Map failureDiffs = {}; - for (final String expectation in testExpectations) { - final List goldenBytes = await skiaClient.getImageBytes(expectation); + final List goldenBytes = await skiaClient.getImageBytes(testExpectation); - result = await GoldenFileComparator.compareLists( - imageBytes, - goldenBytes, - ); + result = await GoldenFileComparator.compareLists( + imageBytes, + goldenBytes, + ); - if (result.passed) { - return true; - } - failureDiffs[expectation] = result; - } + if (result.passed) + return true; - for (final MapEntry entry in failureDiffs.entries) { - if (await skiaClient.isValidDigestForExpectation(entry.key, golden.path)) - generateFailureOutput(entry.value, golden, basedir, key: entry.key); - } + generateFailureOutput(result, golden, basedir); return false; } } diff --git a/packages/flutter_goldens/test/flutter_goldens_test.dart b/packages/flutter_goldens/test/flutter_goldens_test.dart index 1177b87767..98c399c3b3 100644 --- a/packages/flutter_goldens/test/flutter_goldens_test.dart +++ b/packages/flutter_goldens/test/flutter_goldens_test.dart @@ -250,134 +250,128 @@ void main() { ); }); + test('Creates traceID correctly', () { + String traceID; + + // On Cirrus + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + 'CIRRUS_CI' : 'true', + 'CIRRUS_TASK_ID' : '8885996262141582672', + 'CIRRUS_PR' : '49815', + }, + operatingSystem: 'macos' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: mockHttpClient, + ci: ContinuousIntegrationEnvironment.cirrus, + ); + + traceID = skiaClient.getTraceID('flutter.golden.1'); + + expect( + traceID, + equals(',CI=cirrus,Platform=macos,name=flutter.golden.1,source_type=flutter,'), + ); + + // On Luci + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + 'SWARMING_TASK_ID' : '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB' : 'refs/pull/49815/head', + }, + operatingSystem: 'linux' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: mockHttpClient, + ci: ContinuousIntegrationEnvironment.luci, + ); + + traceID = skiaClient.getTraceID('flutter.golden.1'); + + expect( + traceID, + equals(',CI=luci,Platform=linux,name=flutter.golden.1,source_type=flutter,'), + ); + + // Browser + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + 'GOLDCTL' : 'goldctl', + 'SWARMING_TASK_ID' : '4ae997b50dfd4d11', + 'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672', + 'GOLD_TRYJOB' : 'refs/pull/49815/head', + 'FLUTTER_TEST_BROWSER' : 'chrome', + }, + operatingSystem: 'linux' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: mockHttpClient, + ci: ContinuousIntegrationEnvironment.luci, + ); + + traceID = skiaClient.getTraceID('flutter.golden.1'); + + expect( + traceID, + equals(',Browser=chrome,CI=luci,Platform=linux,name=flutter.golden.1,source_type=flutter,'), + ); + + // Locally - should defer to luci traceID + platform = FakePlatform( + environment: { + 'FLUTTER_ROOT': _kFlutterRoot, + }, + operatingSystem: 'macos' + ); + + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: mockHttpClient, + ci: ContinuousIntegrationEnvironment.luci, + ); + + traceID = skiaClient.getTraceID('flutter.golden.1'); + + expect( + traceID, + equals(',CI=luci,Platform=macos,name=flutter.golden.1,source_type=flutter,'), + ); + }); + group('Request Handling', () { String testName; String pullRequestNumber; String expectation; - Uri url; - MockHttpClientRequest mockHttpRequest; setUp(() { testName = 'flutter.golden_test.1.png'; pullRequestNumber = '1234'; expectation = '55109a4bed52acc780530f7a9aeff6c0'; - mockHttpRequest = MockHttpClientRequest(); - }); - - test('validates SkiaDigest', () { - final Map skiaJson = json.decode(digestResponseTemplate()) as Map; - final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map); - expect( - digest.isValid( - platform, - 'flutter.golden_test.1', - expectation, - ), - isTrue, - ); - }); - - test('invalidates bad SkiaDigest - platform', () { - final Map skiaJson = json.decode( - digestResponseTemplate(platform: 'linux'), - ) as Map; - final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map); - expect( - digest.isValid( - platform, - 'flutter.golden_test.1', - expectation, - ), - isFalse, - ); - }); - - test('invalidates bad SkiaDigest - test name', () { - final Map skiaJson = json.decode( - digestResponseTemplate(testName: 'flutter.golden_test.2'), - ) as Map; - final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map); - expect( - digest.isValid( - platform, - 'flutter.golden_test.1', - expectation, - ), - isFalse, - ); - }); - - test('invalidates bad SkiaDigest - expectation', () { - final Map skiaJson = json.decode( - digestResponseTemplate(expectation: '1deg543sf645erg44awqcc78'), - ) as Map; - final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map); - expect( - digest.isValid( - platform, - 'flutter.golden_test.1', - expectation, - ), - isFalse, - ); - }); - - test('invalidates bad SkiaDigest - status', () { - final Map skiaJson = json.decode( - digestResponseTemplate(status: 'negative'), - ) as Map; - final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map); - expect( - digest.isValid( - platform, - 'flutter.golden_test.1', - expectation, - ), - isFalse, - ); - }); - - test('sets up expectations', () async { - url = Uri.parse('https://flutter-gold.skia.org/json/expectations/commit/HEAD'); - final MockHttpClientResponse mockHttpResponse = MockHttpClientResponse( - utf8.encode(rawExpectationsTemplate()) - ); - when(mockHttpClient.getUrl(url)) - .thenAnswer((_) => Future.value(mockHttpRequest)); - when(mockHttpRequest.close()) - .thenAnswer((_) => Future.value(mockHttpResponse)); - - await skiaClient.getExpectations(); - expect(skiaClient.expectations, isNotNull); - expect( - skiaClient.expectations['flutter.golden_test.1'], - contains(expectation), - ); - }); - - test('sets up expectations with temporary key', () async { - url = Uri.parse('https://flutter-gold.skia.org/json/expectations/commit/HEAD'); - final MockHttpClientResponse mockHttpResponse = MockHttpClientResponse( - utf8.encode(rawExpectationsTemplateWithTemporaryKey()) - ); - when(mockHttpClient.getUrl(url)) - .thenAnswer((_) => Future.value(mockHttpRequest)); - when(mockHttpRequest.close()) - .thenAnswer((_) => Future.value(mockHttpResponse)); - - await skiaClient.getExpectations(); - expect(skiaClient.expectations, isNotNull); - expect( - skiaClient.expectations['flutter.golden_test.1'], - contains(expectation), - ); - }); - - test('detects invalid digests SkiaDigest', () { - const String testName = 'flutter.golden_test.2'; - final Map skiaJson = json.decode(digestResponseTemplate()) as Map; - final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map); - expect(digest.isValid(platform, testName, expectation), isFalse); }); test('image bytes are processed properly', () async { @@ -511,50 +505,6 @@ void main() { ); }); }); - - group('digest parsing', () { - Uri url; - MockHttpClientRequest mockHttpRequest; - MockHttpClientResponse mockHttpResponse; - - setUp(() { - url = Uri.parse( - 'https://flutter-gold.skia.org/json/details?' - 'test=flutter.golden_test.1&digest=$expectation' - ); - mockHttpRequest = MockHttpClientRequest(); - when(mockHttpClient.getUrl(url)) - .thenAnswer((_) => Future.value(mockHttpRequest)); - }); - - test('succeeds when valid', () async { - mockHttpResponse = MockHttpClientResponse(utf8.encode(digestResponseTemplate())); - when(mockHttpRequest.close()) - .thenAnswer((_) => Future.value(mockHttpResponse)); - expect( - await skiaClient.isValidDigestForExpectation( - expectation, - testName, - ), - isTrue, - ); - }); - - test('fails when invalid', () async { - mockHttpResponse = MockHttpClientResponse(utf8.encode( - digestResponseTemplate(platform: 'linux') - )); - when(mockHttpRequest.close()) - .thenAnswer((_) => Future.value(mockHttpResponse)); - expect( - await skiaClient.isValidDigestForExpectation( - expectation, - testName, - ), - isFalse, - ); - }); - }); }); }); @@ -838,8 +788,6 @@ void main() { ); when(mockSkiaClient.cleanTestName('library.flutter.golden_test.1.png')) .thenReturn('flutter.golden_test.1'); - when(mockSkiaClient.expectations) - .thenReturn(expectationsTemplate()); }); test('fromDefaultComparator chooses correct comparator', () async { @@ -852,6 +800,9 @@ void main() { test('comparison passes test that is ignored for this PR', () async { when(mockSkiaClient.imgtestCheck(any, any)) .thenAnswer((_) => Future.value(false)); + when(mockSkiaClient.getExpectationForTest('flutter.golden_test.1')) + .thenAnswer((_) => Future.value('123456789abc')); + when(mockSkiaClient.ci).thenReturn(ContinuousIntegrationEnvironment.cirrus); when(mockSkiaClient.testIsIgnoredForPullRequest( '1234', 'library.flutter.golden_test.1.png', @@ -867,8 +818,11 @@ void main() { }); test('fails test that is not ignored', () async { - when(mockSkiaClient.getImageBytes('55109a4bed52acc780530f7a9aeff6c0')) - .thenAnswer((_) => Future>.value(_kTestPngBytes)); + when(mockSkiaClient.imgtestCheck(any, any)) + .thenAnswer((_) => Future.value(false)); + when(mockSkiaClient.getExpectationForTest('flutter.golden_test.1')) + .thenAnswer((_) => Future.value('123456789abc')); + when(mockSkiaClient.ci).thenReturn(ContinuousIntegrationEnvironment.cirrus); when(mockSkiaClient.testIsIgnoredForPullRequest( '1234', 'library.flutter.golden_test.1.png', @@ -947,17 +901,12 @@ void main() { ), ); + when(mockSkiaClient.getExpectationForTest('flutter.golden_test.1')) + .thenAnswer((_) => Future.value('55109a4bed52acc780530f7a9aeff6c0')); when(mockSkiaClient.getImageBytes('55109a4bed52acc780530f7a9aeff6c0')) .thenAnswer((_) => Future>.value(_kTestPngBytes)); - when(mockSkiaClient.expectations) - .thenReturn(expectationsTemplate()); when(mockSkiaClient.cleanTestName('library.flutter.golden_test.1.png')) .thenReturn('flutter.golden_test.1'); - when(mockSkiaClient.isValidDigestForExpectation( - '55109a4bed52acc780530f7a9aeff6c0', - 'library.flutter.golden_test.1.png', - )) - .thenAnswer((_) => Future.value(false)); }); test('passes when bytes match', () async { @@ -985,11 +934,6 @@ void main() { test('compare properly awaits validation & output before failing.', () async { final Completer completer = Completer(); - when(mockSkiaClient.isValidDigestForExpectation( - '55109a4bed52acc780530f7a9aeff6c0', - 'library.flutter.golden_test.1.png', - )) - .thenAnswer((_) => completer.future); final Future result = comparator.compare( Uint8List.fromList(_kFailPngBytes), Uri.parse('flutter.golden_test.1.png'), @@ -1009,7 +953,7 @@ void main() { when(mockDirectory.existsSync()).thenReturn(true); when(mockDirectory.uri).thenReturn(Uri.parse('/flutter')); - when(mockSkiaClient.getExpectations()) + when(mockSkiaClient.getExpectationForTest(any)) .thenAnswer((_) => throw const OSError("Can't reach Gold")); FlutterGoldenFileComparator comparator = await FlutterLocalFileComparator.fromDefaultComparator( platform, @@ -1018,7 +962,7 @@ void main() { ); expect(comparator.runtimeType, FlutterSkippingFileComparator); - when(mockSkiaClient.getExpectations()) + when(mockSkiaClient.getExpectationForTest(any)) .thenAnswer((_) => throw const SocketException("Can't reach Gold")); comparator = await FlutterLocalFileComparator.fromDefaultComparator( platform, diff --git a/packages/flutter_goldens/test/json_templates.dart b/packages/flutter_goldens/test/json_templates.dart index dda77532c8..2def0485ef 100644 --- a/packages/flutter_goldens/test/json_templates.dart +++ b/packages/flutter_goldens/test/json_templates.dart @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -/// JSON template for the contents of the auth_opt.json file created by goldctl. +/// Json response template for the contents of the auth_opt.json file created by +/// goldctl. String authTemplate({ bool gsutil = false, }) { @@ -15,196 +16,6 @@ String authTemplate({ '''; } -/// JSON response template for Skia Gold expectations request: -/// https://flutter-gold.skia.org/json/expectations/commit/HEAD -String rawExpectationsTemplate() { - return ''' - { - "md5": "a7489b00e03a1846e43500b7c14dd7b0", - "master": { - "flutter.golden_test.1": { - "55109a4bed52acc780530f7a9aeff6c0": 1 - }, - "flutter.golden_test.3": { - "87cb35131e6ad4b57d4d09d59ae743c3": 1, - "dc94eb2c39c0c8ae11a4efd090b72f94": 1, - "f2583c9003978a06b7888878bdc089e2": 1 - }, - "flutter.golden_test.2": { - "eb03a5e3114c9ecad5e4f1178f285a49": 1, - "f14631979de24fca6e14ad247d5f2bd6": 1 - } - } - } - '''; -} - -/// Decoded json response template for Skia Gold expectations request: -/// https://flutter-gold.skia.org/json/expectations/commit/HEAD -Map> expectationsTemplate() { - return >{ - 'flutter.golden_test.1': [ - '55109a4bed52acc780530f7a9aeff6c0' - ], - 'flutter.golden_test.3': [ - '87cb35131e6ad4b57d4d09d59ae743c3', - 'dc94eb2c39c0c8ae11a4efd090b72f94', - 'f2583c9003978a06b7888878bdc089e2', - ], - 'flutter.golden_test.2': [ - 'eb03a5e3114c9ecad5e4f1178f285a49', - 'f14631979de24fca6e14ad247d5f2bd6', - ], - }; -} - -/// Same as [rawExpectationsTemplate] but with the temporary key. -String rawExpectationsTemplateWithTemporaryKey() { - return ''' - { - "md5": "a7489b00e03a1846e43500b7c14dd7b0", - "master_str": { - "flutter.golden_test.1": { - "55109a4bed52acc780530f7a9aeff6c0": 1 - }, - "flutter.golden_test.3": { - "87cb35131e6ad4b57d4d09d59ae743c3": 1, - "dc94eb2c39c0c8ae11a4efd090b72f94": 1, - "f2583c9003978a06b7888878bdc089e2": 1 - }, - "flutter.golden_test.2": { - "eb03a5e3114c9ecad5e4f1178f285a49": 1, - "f14631979de24fca6e14ad247d5f2bd6": 1 - } - } - } - '''; -} - -/// Json response template for Skia Gold digest request: -/// https://flutter-gold.skia.org/json/details?test=[testName]&digest=[expectation] -String digestResponseTemplate({ - String testName = 'flutter.golden_test.1', - String expectation = '55109a4bed52acc780530f7a9aeff6c0', - String platform = 'macos', - String status = 'positive', -}) { - return ''' - { - "digest": { - "test": "$testName", - "digest": "$expectation", - "status": "$status", - "paramset": { - "Platform": [ - "$platform" - ], - "ext": [ - "png" - ], - "name": [ - "$testName" - ], - "source_type": [ - "flutter" - ] - }, - "traces": { - "tileSize": 200, - "traces": [ - { - "data": [ - { - "x": 0, - "y": 0, - "s": 0 - }, - { - "x": 1, - "y": 0, - "s": 0 - }, - { - "x": 199, - "y": 0, - "s": 0 - } - ], - "label": ",Platform=$platform,name=$testName,source_type=flutter,", - "params": { - "Platform": "$platform", - "ext": "png", - "name": "$testName", - "source_type": "flutter" - } - } - ], - "digests": [ - { - "digest": "$expectation", - "status": "$status" - } - ] - }, - "closestRef": "pos", - "refDiffs": { - "neg": null, - "pos": { - "numDiffPixels": 999, - "pixelDiffPercent": 0.4995, - "maxRGBADiffs": [ - 86, - 86, - 86, - 0 - ], - "dimDiffer": false, - "diffs": { - "combined": 0.381955, - "percent": 0.4995, - "pixel": 999 - }, - "digest": "aa748136c70cefdda646df5be0ae189d", - "status": "positive", - "paramset": { - "Platform": [ - "$platform" - ], - "ext": [ - "png" - ], - "name": [ - "$testName" - ], - "source_type": [ - "flutter" - ] - }, - "n": 197 - } - } - }, - "commits": [ - { - "commit_time": 1568069344, - "hash": "399bb04e2de41665320d3c888f40af6d8bc734a2", - "author": "Contributor A (contributorA@getMail.com)" - }, - { - "commit_time": 1568078053, - "hash": "0f365d3add253a65e5e5af1024f56c6169bf9739", - "author": "Contributor B (contributorB@getMail.com)" - }, - { - "commit_time": 1569353925, - "hash": "81e693a7fe3b808cc9ae2bb3a2cbe404e67ec773", - "author": "Contributor C (contributorC@getMail.com)" - } - ] -} - '''; -} - /// Json response template for Skia Gold ignore request: /// https://flutter-gold.skia.org/json/ignores String ignoreResponseTemplate({ diff --git a/packages/flutter_goldens_client/lib/skia_client.dart b/packages/flutter_goldens_client/lib/skia_client.dart index 54a76d401e..4702ea1f04 100644 --- a/packages/flutter_goldens_client/lib/skia_client.dart +++ b/packages/flutter_goldens_client/lib/skia_client.dart @@ -73,15 +73,6 @@ class SkiaGoldClient { /// be null. final Directory workDirectory; - /// A map of known golden file tests and their associated positive image - /// hashes. - /// - /// This is set and used by the [FlutterLocalFileComparator] and the - /// [_UnauthorizedFlutterPreSubmitComparator] to test against golden masters - /// maintained in the Flutter Gold dashboard. - Map> get expectations => _expectations; - late Map> _expectations; - /// The local [Directory] where the Flutter repository is hosted. /// /// Uses the [fs] file system. @@ -421,15 +412,15 @@ class SkiaGoldClient { return result.exitCode == 0; } - /// Requests and sets the [_expectations] known to Flutter Gold at head. - Future getExpectations() async { - _expectations = >{}; + /// Returns the latest positive digest for the given test known to Flutter + /// Gold at head. + Future getExpectationForTest(String testName) async { + late String? expectation; + final String traceID = getTraceID(testName); await io.HttpOverrides.runWithHttpOverrides>(() async { final Uri requestForExpectations = Uri.parse( - 'https://flutter-gold.skia.org/json/expectations/commit/HEAD' + 'https://flutter-gold.skia.org/json/latestpositivedigest/$traceID' ); - const String mainKey = 'master'; - const String temporaryKey = 'master_str'; late String rawResponse; try { final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations); @@ -438,13 +429,7 @@ class SkiaGoldClient { final dynamic jsonResponse = json.decode(rawResponse); if (jsonResponse is! Map) throw const FormatException('Skia gold expectations do not match expected format.'); - final Map? skiaJson = (jsonResponse[mainKey] ?? jsonResponse[temporaryKey]) as Map?; - if (skiaJson == null) - throw FormatException('Skia gold expectations are missing the "$mainKey" key (and also doesn\'t have "$temporaryKey")! Available keys: ${jsonResponse.keys.join(", ")}'); - skiaJson.forEach((String key, dynamic value) { - final Map hashesMap = value as Map; - _expectations[key] = hashesMap.keys.toList(); - }); + expectation = jsonResponse['digest'] as String?; } on FormatException catch (error) { print( 'Formatting error detected requesting expectations from Flutter Gold.\n' @@ -457,6 +442,7 @@ class SkiaGoldClient { }, SkiaGoldHttpOverrides(), ); + return expectation; } /// Returns a list of bytes representing the golden image retrieved from the @@ -548,44 +534,6 @@ class SkiaGoldClient { return ignoreIsActive; } - /// The [_expectations] retrieved from Flutter Gold do not include the - /// parameters of the given test. This function queries the Flutter Gold - /// details api to determine if the given expectation for a test matches the - /// configuration of the executing machine. - Future isValidDigestForExpectation(String expectation, String testName) async { - bool isValid = false; - testName = cleanTestName(testName); - late String rawResponse; - await io.HttpOverrides.runWithHttpOverrides>(() async { - final Uri requestForDigest = Uri.parse( - 'https://flutter-gold.skia.org/json/details?test=$testName&digest=$expectation' - ); - - try { - final io.HttpClientRequest request = await httpClient.getUrl(requestForDigest); - final io.HttpClientResponse response = await request.close(); - rawResponse = await utf8.decodeStream(response); - final Map skiaJson = json.decode(rawResponse) as Map; - final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map); - isValid = digest.isValid(platform, testName, expectation); - - } on FormatException catch(_) { - if (rawResponse.contains('stream timeout')) { - final StringBuffer buf = StringBuffer() - ..writeln("Stream timeout on Gold's /details api."); - throw Exception(buf.toString()); - } else { - print('Formatting error detected requesting /ignores from Flutter Gold.' - '\nrawResponse: $rawResponse'); - rethrow; - } - } - }, - SkiaGoldHttpOverrides(), - ); - return isValid; - } - /// Returns the current commit hash of the Flutter repository. Future _getCurrentCommit() async { if (!_flutterRoot.existsSync()) { @@ -669,55 +617,26 @@ class SkiaGoldClient { '--jobid', jobId, ]; } + + /// Returns a trace id based on the current testing environment to lookup + /// the latest positive digest on Flutter Gold. + /// + /// Trace IDs are case sensitive and should be in alphabetical order for the + /// keys, followed by the rest of the paramset, also in alphabetical order. + /// There should also be leading and trailing commas. + /// + /// Example TraceID for Flutter Gold: + /// ',CI=cirrus,Platform=linux,name=cupertino.activityIndicator.inprogress.1.0,source_type=flutter,' + String getTraceID(String testName) { + // If we are not in a CI environment, fallback on luci. + return '${platform.environment[_kTestBrowserKey] == null ? ',' : ',Browser=${platform.environment[_kTestBrowserKey]},'}' + 'CI=${ci == ContinuousIntegrationEnvironment.none ? 'luci' : ci.toString().split('.').last},' + 'Platform=${platform.operatingSystem},' + 'name=$testName,' + 'source_type=flutter,'; + } + } /// Used to make HttpRequests during testing. class SkiaGoldHttpOverrides extends io.HttpOverrides {} - -/// A digest returned from a request to the Flutter Gold dashboard. -class SkiaGoldDigest { - const SkiaGoldDigest({ - required this.imageHash, - required this.paramSet, - required this.testName, - required this.status, - }); - - /// Create a digest from requested JSON. - factory SkiaGoldDigest.fromJson(Map json) { - return SkiaGoldDigest( - imageHash: json['digest'] as String, - paramSet: - Map.from( - json['refDiffs']['pos']['paramset'] as Map? ?? - >{ - 'Platform': [], - 'Browser' : [], - }), - testName: json['test'] as String, - status: json['status'] as String, - ); - } - - /// Unique identifier for the image associated with the digest. - final String imageHash; - - /// Parameter set for the given test, e.g. Platform : Windows. - final Map paramSet; - - /// Test name associated with the digest, e.g. positive or un-triaged. - final String testName; - - /// Status of the given digest, e.g. positive or un-triaged. - final String status; - - /// Validates a given digest against the current testing conditions. - bool isValid(Platform platform, String name, String expectation) { - return imageHash == expectation - && (paramSet['Platform'] as List/*!*/).contains(platform.operatingSystem) - && (platform.environment[_kTestBrowserKey] == null - || paramSet['Browser'] == platform.environment[_kTestBrowserKey]) - && testName == name - && status == 'positive'; - } -}