firka/flutter
3
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update Gold for new endpoint (#64982)

Browse Source
This commit is contained in:
Kate Lovett
2020-09-02 16:35:06 -07:00
committed by GitHub
parent 37de94d7d2
commit ee69eebfbd
4 changed files with 193 additions and 514 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
packages/flutter_goldens/lib/flutter_goldens.dart
View File

@@ -2,10 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:io' as io;
import 'dart:math' as math;
import 'dart:typed_data';
import 'dart:async' show FutureOr;
import 'dart:io' as io show OSError, SocketException;
import 'dart:math' as math show Random;
import 'dart:typed_data' show Uint8List;
import 'package:file/file.dart';
import 'package:file/local.dart';
@@ -449,13 +449,10 @@ class _UnauthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileCompar
if (await skiaClient.imgtestCheck(golden.path, goldenFile))
return true;
// We do not have a matching image, so we need to check a few things
// manually. We wait until this point to do this work so request traffic
// low.
skiaClient.getExpectations();
// We do not have a matching image hash, so we need to check manually.
final String testName = skiaClient.cleanTestName(golden.path);
final List<String>? testExpectations = skiaClient.expectations[testName];
if (testExpectations == null) {
final String? testExpectation = await skiaClient.getExpectationForTest(testName);
if (testExpectation == null) {
// This is a new test.
print('No expectations provided by Skia Gold for test: $golden. '
'This may be a new test. If this is an unexpected result, check '
@@ -466,11 +463,26 @@ class _UnauthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileCompar
// Contributors without the proper permissions to execute a tryjob can make
// a golden file change through Gold's ignore feature instead.
String? pullRequest;
switch(skiaClient.ci) {
case ContinuousIntegrationEnvironment.cirrus:
pullRequest = platform.environment['CIRRUS_PR']!;
break;
case ContinuousIntegrationEnvironment.luci:
final List<String> refs = platform.environment['GOLD_TRYJOB']!.split('/');
pullRequest = refs[refs.length - 2];
break;
case ContinuousIntegrationEnvironment.none:
pullRequest = '';
break;
}
final bool ignoreResult = await skiaClient.testIsIgnoredForPullRequest(
platform.environment['CIRRUS_PR'] ?? '',
pullRequest,
golden.path,
);
// If true, this is an intended change.
// If true, this is an intended change and is being handled on the Flutter
// Gold dashboard: https://flutter-gold.skia.org/ignores
return ignoreResult;
}
}
@@ -479,8 +491,7 @@ class _UnauthorizedFlutterPreSubmitComparator extends FlutterPreSubmitFileCompar
/// golden file tests.
///
/// Currently, this comparator is used in some Cirrus test shards and Luci
/// environments, as well as when an internet connection is not available for
/// contacting Gold.
/// environments.
///
/// See also:
///
@@ -608,9 +619,9 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC
}
goldens ??= SkiaGoldClient(baseDirectory, ci: ContinuousIntegrationEnvironment.none);
try {
await goldens.getExpectations();
// Check if we can reach Gold.
await goldens.getExpectationForTest('');
} on io.OSError catch (_) {
return FlutterSkippingFileComparator(
baseDirectory.uri,
@@ -634,8 +645,10 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC
Future<bool> compare(Uint8List imageBytes, Uri golden) async {
golden = _addPrefix(golden);
final String testName = skiaClient.cleanTestName(golden.path);
final List<String>? testExpectations = skiaClient.expectations[testName];
if (testExpectations == null) {
late String? testExpectation;
testExpectation = await skiaClient.getExpectationForTest(testName);
if (testExpectation == null) {
// There is no baseline for this test
print('No expectations provided by Skia Gold for test: $golden. '
'This may be a new test. If this is an unexpected result, check '
@@ -647,25 +660,17 @@ class FlutterLocalFileComparator extends FlutterGoldenFileComparator with LocalC
}
ComparisonResult result;
final Map<String, ComparisonResult> failureDiffs = <String, ComparisonResult>{};
for (final String expectation in testExpectations) {
final List<int> goldenBytes = await skiaClient.getImageBytes(expectation);
final List<int> goldenBytes = await skiaClient.getImageBytes(testExpectation);
result = await GoldenFileComparator.compareLists(
imageBytes,
goldenBytes,
);
result = await GoldenFileComparator.compareLists(
imageBytes,
goldenBytes,
);
if (result.passed) {
return true;
}
failureDiffs[expectation] = result;
}
if (result.passed)
return true;
for (final MapEntry<String, ComparisonResult> entry in failureDiffs.entries) {
if (await skiaClient.isValidDigestForExpectation(entry.key, golden.path))
generateFailureOutput(entry.value, golden, basedir, key: entry.key);
}
generateFailureOutput(result, golden, basedir);
return false;
}
}

306
packages/flutter_goldens/test/flutter_goldens_test.dart
View File

@@ -250,134 +250,128 @@ void main() {
);
});
test('Creates traceID correctly', () {
String traceID;
// On Cirrus
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl',
'CIRRUS_CI' : 'true',
'CIRRUS_TASK_ID' : '8885996262141582672',
'CIRRUS_PR' : '49815',
},
operatingSystem: 'macos'
);
skiaClient = SkiaGoldClient(
workDirectory,
fs: fs,
process: process,
platform: platform,
httpClient: mockHttpClient,
ci: ContinuousIntegrationEnvironment.cirrus,
);
traceID = skiaClient.getTraceID('flutter.golden.1');
expect(
traceID,
equals(',CI=cirrus,Platform=macos,name=flutter.golden.1,source_type=flutter,'),
);
// On Luci
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl',
'SWARMING_TASK_ID' : '4ae997b50dfd4d11',
'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672',
'GOLD_TRYJOB' : 'refs/pull/49815/head',
},
operatingSystem: 'linux'
);
skiaClient = SkiaGoldClient(
workDirectory,
fs: fs,
process: process,
platform: platform,
httpClient: mockHttpClient,
ci: ContinuousIntegrationEnvironment.luci,
);
traceID = skiaClient.getTraceID('flutter.golden.1');
expect(
traceID,
equals(',CI=luci,Platform=linux,name=flutter.golden.1,source_type=flutter,'),
);
// Browser
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
'GOLDCTL' : 'goldctl',
'SWARMING_TASK_ID' : '4ae997b50dfd4d11',
'LOGDOG_STREAM_PREFIX' : 'buildbucket/cr-buildbucket.appspot.com/8885996262141582672',
'GOLD_TRYJOB' : 'refs/pull/49815/head',
'FLUTTER_TEST_BROWSER' : 'chrome',
},
operatingSystem: 'linux'
);
skiaClient = SkiaGoldClient(
workDirectory,
fs: fs,
process: process,
platform: platform,
httpClient: mockHttpClient,
ci: ContinuousIntegrationEnvironment.luci,
);
traceID = skiaClient.getTraceID('flutter.golden.1');
expect(
traceID,
equals(',Browser=chrome,CI=luci,Platform=linux,name=flutter.golden.1,source_type=flutter,'),
);
// Locally - should defer to luci traceID
platform = FakePlatform(
environment: <String, String>{
'FLUTTER_ROOT': _kFlutterRoot,
},
operatingSystem: 'macos'
);
skiaClient = SkiaGoldClient(
workDirectory,
fs: fs,
process: process,
platform: platform,
httpClient: mockHttpClient,
ci: ContinuousIntegrationEnvironment.luci,
);
traceID = skiaClient.getTraceID('flutter.golden.1');
expect(
traceID,
equals(',CI=luci,Platform=macos,name=flutter.golden.1,source_type=flutter,'),
);
});
group('Request Handling', () {
String testName;
String pullRequestNumber;
String expectation;
Uri url;
MockHttpClientRequest mockHttpRequest;
setUp(() {
testName = 'flutter.golden_test.1.png';
pullRequestNumber = '1234';
expectation = '55109a4bed52acc780530f7a9aeff6c0';
mockHttpRequest = MockHttpClientRequest();
});
test('validates SkiaDigest', () {
final Map<String, dynamic> skiaJson = json.decode(digestResponseTemplate()) as Map<String, dynamic>;
final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map<String, dynamic>);
expect(
digest.isValid(
platform,
'flutter.golden_test.1',
expectation,
),
isTrue,
);
});
test('invalidates bad SkiaDigest - platform', () {
final Map<String, dynamic> skiaJson = json.decode(
digestResponseTemplate(platform: 'linux'),
) as Map<String, dynamic>;
final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map<String, dynamic>);
expect(
digest.isValid(
platform,
'flutter.golden_test.1',
expectation,
),
isFalse,
);
});
test('invalidates bad SkiaDigest - test name', () {
final Map<String, dynamic> skiaJson = json.decode(
digestResponseTemplate(testName: 'flutter.golden_test.2'),
) as Map<String, dynamic>;
final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map<String, dynamic>);
expect(
digest.isValid(
platform,
'flutter.golden_test.1',
expectation,
),
isFalse,
);
});
test('invalidates bad SkiaDigest - expectation', () {
final Map<String, dynamic> skiaJson = json.decode(
digestResponseTemplate(expectation: '1deg543sf645erg44awqcc78'),
) as Map<String, dynamic>;
final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map<String, dynamic>);
expect(
digest.isValid(
platform,
'flutter.golden_test.1',
expectation,
),
isFalse,
);
});
test('invalidates bad SkiaDigest - status', () {
final Map<String, dynamic> skiaJson = json.decode(
digestResponseTemplate(status: 'negative'),
) as Map<String, dynamic>;
final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map<String, dynamic>);
expect(
digest.isValid(
platform,
'flutter.golden_test.1',
expectation,
),
isFalse,
);
});
test('sets up expectations', () async {
url = Uri.parse('https://flutter-gold.skia.org/json/expectations/commit/HEAD');
final MockHttpClientResponse mockHttpResponse = MockHttpClientResponse(
utf8.encode(rawExpectationsTemplate())
);
when(mockHttpClient.getUrl(url))
.thenAnswer((_) => Future<MockHttpClientRequest>.value(mockHttpRequest));
when(mockHttpRequest.close())
.thenAnswer((_) => Future<MockHttpClientResponse>.value(mockHttpResponse));
await skiaClient.getExpectations();
expect(skiaClient.expectations, isNotNull);
expect(
skiaClient.expectations['flutter.golden_test.1'],
contains(expectation),
);
});
test('sets up expectations with temporary key', () async {
url = Uri.parse('https://flutter-gold.skia.org/json/expectations/commit/HEAD');
final MockHttpClientResponse mockHttpResponse = MockHttpClientResponse(
utf8.encode(rawExpectationsTemplateWithTemporaryKey())
);
when(mockHttpClient.getUrl(url))
.thenAnswer((_) => Future<MockHttpClientRequest>.value(mockHttpRequest));
when(mockHttpRequest.close())
.thenAnswer((_) => Future<MockHttpClientResponse>.value(mockHttpResponse));
await skiaClient.getExpectations();
expect(skiaClient.expectations, isNotNull);
expect(
skiaClient.expectations['flutter.golden_test.1'],
contains(expectation),
);
});
test('detects invalid digests SkiaDigest', () {
const String testName = 'flutter.golden_test.2';
final Map<String, dynamic> skiaJson = json.decode(digestResponseTemplate()) as Map<String, dynamic>;
final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map<String, dynamic>);
expect(digest.isValid(platform, testName, expectation), isFalse);
});
test('image bytes are processed properly', () async {
@@ -511,50 +505,6 @@ void main() {
);
});
});
group('digest parsing', () {
Uri url;
MockHttpClientRequest mockHttpRequest;
MockHttpClientResponse mockHttpResponse;
setUp(() {
url = Uri.parse(
'https://flutter-gold.skia.org/json/details?'
'test=flutter.golden_test.1&digest=$expectation'
);
mockHttpRequest = MockHttpClientRequest();
when(mockHttpClient.getUrl(url))
.thenAnswer((_) => Future<MockHttpClientRequest>.value(mockHttpRequest));
});
test('succeeds when valid', () async {
mockHttpResponse = MockHttpClientResponse(utf8.encode(digestResponseTemplate()));
when(mockHttpRequest.close())
.thenAnswer((_) => Future<MockHttpClientResponse>.value(mockHttpResponse));
expect(
await skiaClient.isValidDigestForExpectation(
expectation,
testName,
),
isTrue,
);
});
test('fails when invalid', () async {
mockHttpResponse = MockHttpClientResponse(utf8.encode(
digestResponseTemplate(platform: 'linux')
));
when(mockHttpRequest.close())
.thenAnswer((_) => Future<MockHttpClientResponse>.value(mockHttpResponse));
expect(
await skiaClient.isValidDigestForExpectation(
expectation,
testName,
),
isFalse,
);
});
});
});
});
@@ -838,8 +788,6 @@ void main() {
);
when(mockSkiaClient.cleanTestName('library.flutter.golden_test.1.png'))
.thenReturn('flutter.golden_test.1');
when(mockSkiaClient.expectations)
.thenReturn(expectationsTemplate());
});
test('fromDefaultComparator chooses correct comparator', () async {
@@ -852,6 +800,9 @@ void main() {
test('comparison passes test that is ignored for this PR', () async {
when(mockSkiaClient.imgtestCheck(any, any))
.thenAnswer((_) => Future<bool>.value(false));
when(mockSkiaClient.getExpectationForTest('flutter.golden_test.1'))
.thenAnswer((_) => Future<String>.value('123456789abc'));
when(mockSkiaClient.ci).thenReturn(ContinuousIntegrationEnvironment.cirrus);
when(mockSkiaClient.testIsIgnoredForPullRequest(
'1234',
'library.flutter.golden_test.1.png',
@@ -867,8 +818,11 @@ void main() {
});
test('fails test that is not ignored', () async {
when(mockSkiaClient.getImageBytes('55109a4bed52acc780530f7a9aeff6c0'))
.thenAnswer((_) => Future<List<int>>.value(_kTestPngBytes));
when(mockSkiaClient.imgtestCheck(any, any))
.thenAnswer((_) => Future<bool>.value(false));
when(mockSkiaClient.getExpectationForTest('flutter.golden_test.1'))
.thenAnswer((_) => Future<String>.value('123456789abc'));
when(mockSkiaClient.ci).thenReturn(ContinuousIntegrationEnvironment.cirrus);
when(mockSkiaClient.testIsIgnoredForPullRequest(
'1234',
'library.flutter.golden_test.1.png',
@@ -947,17 +901,12 @@ void main() {
),
);
when(mockSkiaClient.getExpectationForTest('flutter.golden_test.1'))
.thenAnswer((_) => Future<String>.value('55109a4bed52acc780530f7a9aeff6c0'));
when(mockSkiaClient.getImageBytes('55109a4bed52acc780530f7a9aeff6c0'))
.thenAnswer((_) => Future<List<int>>.value(_kTestPngBytes));
when(mockSkiaClient.expectations)
.thenReturn(expectationsTemplate());
when(mockSkiaClient.cleanTestName('library.flutter.golden_test.1.png'))
.thenReturn('flutter.golden_test.1');
when(mockSkiaClient.isValidDigestForExpectation(
'55109a4bed52acc780530f7a9aeff6c0',
'library.flutter.golden_test.1.png',
))
.thenAnswer((_) => Future<bool>.value(false));
});
test('passes when bytes match', () async {
@@ -985,11 +934,6 @@ void main() {
test('compare properly awaits validation & output before failing.', () async {
final Completer<bool> completer = Completer<bool>();
when(mockSkiaClient.isValidDigestForExpectation(
'55109a4bed52acc780530f7a9aeff6c0',
'library.flutter.golden_test.1.png',
))
.thenAnswer((_) => completer.future);
final Future<bool> result = comparator.compare(
Uint8List.fromList(_kFailPngBytes),
Uri.parse('flutter.golden_test.1.png'),
@@ -1009,7 +953,7 @@ void main() {
when(mockDirectory.existsSync()).thenReturn(true);
when(mockDirectory.uri).thenReturn(Uri.parse('/flutter'));
when(mockSkiaClient.getExpectations())
when(mockSkiaClient.getExpectationForTest(any))
.thenAnswer((_) => throw const OSError("Can't reach Gold"));
FlutterGoldenFileComparator comparator = await FlutterLocalFileComparator.fromDefaultComparator(
platform,
@@ -1018,7 +962,7 @@ void main() {
);
expect(comparator.runtimeType, FlutterSkippingFileComparator);
when(mockSkiaClient.getExpectations())
when(mockSkiaClient.getExpectationForTest(any))
.thenAnswer((_) => throw const SocketException("Can't reach Gold"));
comparator = await FlutterLocalFileComparator.fromDefaultComparator(
platform,

193
packages/flutter_goldens/test/json_templates.dart
View File

@@ -2,7 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
/// JSON template for the contents of the auth_opt.json file created by goldctl.
/// Json response template for the contents of the auth_opt.json file created by
/// goldctl.
String authTemplate({
bool gsutil = false,
}) {
@@ -15,196 +16,6 @@ String authTemplate({
''';
}
/// JSON response template for Skia Gold expectations request:
/// https://flutter-gold.skia.org/json/expectations/commit/HEAD
String rawExpectationsTemplate() {
return '''
{
"md5": "a7489b00e03a1846e43500b7c14dd7b0",
"master": {
"flutter.golden_test.1": {
"55109a4bed52acc780530f7a9aeff6c0": 1
},
"flutter.golden_test.3": {
"87cb35131e6ad4b57d4d09d59ae743c3": 1,
"dc94eb2c39c0c8ae11a4efd090b72f94": 1,
"f2583c9003978a06b7888878bdc089e2": 1
},
"flutter.golden_test.2": {
"eb03a5e3114c9ecad5e4f1178f285a49": 1,
"f14631979de24fca6e14ad247d5f2bd6": 1
}
}
}
''';
}
/// Decoded json response template for Skia Gold expectations request:
/// https://flutter-gold.skia.org/json/expectations/commit/HEAD
Map<String, List<String>> expectationsTemplate() {
return <String, List<String>>{
'flutter.golden_test.1': <String>[
'55109a4bed52acc780530f7a9aeff6c0'
],
'flutter.golden_test.3': <String>[
'87cb35131e6ad4b57d4d09d59ae743c3',
'dc94eb2c39c0c8ae11a4efd090b72f94',
'f2583c9003978a06b7888878bdc089e2',
],
'flutter.golden_test.2': <String>[
'eb03a5e3114c9ecad5e4f1178f285a49',
'f14631979de24fca6e14ad247d5f2bd6',
],
};
}
/// Same as [rawExpectationsTemplate] but with the temporary key.
String rawExpectationsTemplateWithTemporaryKey() {
return '''
{
"md5": "a7489b00e03a1846e43500b7c14dd7b0",
"master_str": {
"flutter.golden_test.1": {
"55109a4bed52acc780530f7a9aeff6c0": 1
},
"flutter.golden_test.3": {
"87cb35131e6ad4b57d4d09d59ae743c3": 1,
"dc94eb2c39c0c8ae11a4efd090b72f94": 1,
"f2583c9003978a06b7888878bdc089e2": 1
},
"flutter.golden_test.2": {
"eb03a5e3114c9ecad5e4f1178f285a49": 1,
"f14631979de24fca6e14ad247d5f2bd6": 1
}
}
}
''';
}
/// Json response template for Skia Gold digest request:
/// https://flutter-gold.skia.org/json/details?test=[testName]&digest=[expectation]
String digestResponseTemplate({
String testName = 'flutter.golden_test.1',
String expectation = '55109a4bed52acc780530f7a9aeff6c0',
String platform = 'macos',
String status = 'positive',
}) {
return '''
{
"digest": {
"test": "$testName",
"digest": "$expectation",
"status": "$status",
"paramset": {
"Platform": [
"$platform"
],
"ext": [
"png"
],
"name": [
"$testName"
],
"source_type": [
"flutter"
]
},
"traces": {
"tileSize": 200,
"traces": [
{
"data": [
{
"x": 0,
"y": 0,
"s": 0
},
{
"x": 1,
"y": 0,
"s": 0
},
{
"x": 199,
"y": 0,
"s": 0
}
],
"label": ",Platform=$platform,name=$testName,source_type=flutter,",
"params": {
"Platform": "$platform",
"ext": "png",
"name": "$testName",
"source_type": "flutter"
}
}
],
"digests": [
{
"digest": "$expectation",
"status": "$status"
}
]
},
"closestRef": "pos",
"refDiffs": {
"neg": null,
"pos": {
"numDiffPixels": 999,
"pixelDiffPercent": 0.4995,
"maxRGBADiffs": [
86,
86,
86,
0
],
"dimDiffer": false,
"diffs": {
"combined": 0.381955,
"percent": 0.4995,
"pixel": 999
},
"digest": "aa748136c70cefdda646df5be0ae189d",
"status": "positive",
"paramset": {
"Platform": [
"$platform"
],
"ext": [
"png"
],
"name": [
"$testName"
],
"source_type": [
"flutter"
]
},
"n": 197
}
}
},
"commits": [
{
"commit_time": 1568069344,
"hash": "399bb04e2de41665320d3c888f40af6d8bc734a2",
"author": "Contributor A (contributorA@getMail.com)"
},
{
"commit_time": 1568078053,
"hash": "0f365d3add253a65e5e5af1024f56c6169bf9739",
"author": "Contributor B (contributorB@getMail.com)"
},
{
"commit_time": 1569353925,
"hash": "81e693a7fe3b808cc9ae2bb3a2cbe404e67ec773",
"author": "Contributor C (contributorC@getMail.com)"
}
]
}
''';
}
/// Json response template for Skia Gold ignore request:
/// https://flutter-gold.skia.org/json/ignores
String ignoreResponseTemplate({

135
packages/flutter_goldens_client/lib/skia_client.dart
View File

@@ -73,15 +73,6 @@ class SkiaGoldClient {
/// be null.
final Directory workDirectory;
/// A map of known golden file tests and their associated positive image
/// hashes.
///
/// This is set and used by the [FlutterLocalFileComparator] and the
/// [_UnauthorizedFlutterPreSubmitComparator] to test against golden masters
/// maintained in the Flutter Gold dashboard.
Map<String, List<String>> get expectations => _expectations;
late Map<String, List<String>> _expectations;
/// The local [Directory] where the Flutter repository is hosted.
///
/// Uses the [fs] file system.
@@ -421,15 +412,15 @@ class SkiaGoldClient {
return result.exitCode == 0;
}
/// Requests and sets the [_expectations] known to Flutter Gold at head.
Future<void> getExpectations() async {
_expectations = <String, List<String>>{};
/// Returns the latest positive digest for the given test known to Flutter
/// Gold at head.
Future<String?> getExpectationForTest(String testName) async {
late String? expectation;
final String traceID = getTraceID(testName);
await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
final Uri requestForExpectations = Uri.parse(
'https://flutter-gold.skia.org/json/expectations/commit/HEAD'
'https://flutter-gold.skia.org/json/latestpositivedigest/$traceID'
);
const String mainKey = 'master';
const String temporaryKey = 'master_str';
late String rawResponse;
try {
final io.HttpClientRequest request = await httpClient.getUrl(requestForExpectations);
@@ -438,13 +429,7 @@ class SkiaGoldClient {
final dynamic jsonResponse = json.decode(rawResponse);
if (jsonResponse is! Map<String, dynamic>)
throw const FormatException('Skia gold expectations do not match expected format.');
final Map<String, dynamic>? skiaJson = (jsonResponse[mainKey] ?? jsonResponse[temporaryKey]) as Map<String, dynamic>?;
if (skiaJson == null)
throw FormatException('Skia gold expectations are missing the "$mainKey" key (and also doesn\'t have "$temporaryKey")! Available keys: ${jsonResponse.keys.join(", ")}');
skiaJson.forEach((String key, dynamic value) {
final Map<String, dynamic> hashesMap = value as Map<String, dynamic>;
_expectations[key] = hashesMap.keys.toList();
});
expectation = jsonResponse['digest'] as String?;
} on FormatException catch (error) {
print(
'Formatting error detected requesting expectations from Flutter Gold.\n'
@@ -457,6 +442,7 @@ class SkiaGoldClient {
},
SkiaGoldHttpOverrides(),
);
return expectation;
}
/// Returns a list of bytes representing the golden image retrieved from the
@@ -548,44 +534,6 @@ class SkiaGoldClient {
return ignoreIsActive;
}
/// The [_expectations] retrieved from Flutter Gold do not include the
/// parameters of the given test. This function queries the Flutter Gold
/// details api to determine if the given expectation for a test matches the
/// configuration of the executing machine.
Future<bool> isValidDigestForExpectation(String expectation, String testName) async {
bool isValid = false;
testName = cleanTestName(testName);
late String rawResponse;
await io.HttpOverrides.runWithHttpOverrides<Future<void>>(() async {
final Uri requestForDigest = Uri.parse(
'https://flutter-gold.skia.org/json/details?test=$testName&digest=$expectation'
);
try {
final io.HttpClientRequest request = await httpClient.getUrl(requestForDigest);
final io.HttpClientResponse response = await request.close();
rawResponse = await utf8.decodeStream(response);
final Map<String, dynamic> skiaJson = json.decode(rawResponse) as Map<String, dynamic>;
final SkiaGoldDigest digest = SkiaGoldDigest.fromJson(skiaJson['digest'] as Map<String, dynamic>);
isValid = digest.isValid(platform, testName, expectation);
} on FormatException catch(_) {
if (rawResponse.contains('stream timeout')) {
final StringBuffer buf = StringBuffer()
..writeln("Stream timeout on Gold's /details api.");
throw Exception(buf.toString());
} else {
print('Formatting error detected requesting /ignores from Flutter Gold.'
'\nrawResponse: $rawResponse');
rethrow;
}
}
},
SkiaGoldHttpOverrides(),
);
return isValid;
}
/// Returns the current commit hash of the Flutter repository.
Future<String> _getCurrentCommit() async {
if (!_flutterRoot.existsSync()) {
@@ -669,55 +617,26 @@ class SkiaGoldClient {
'--jobid', jobId,
];
}
/// Returns a trace id based on the current testing environment to lookup
/// the latest positive digest on Flutter Gold.
///
/// Trace IDs are case sensitive and should be in alphabetical order for the
/// keys, followed by the rest of the paramset, also in alphabetical order.
/// There should also be leading and trailing commas.
///
/// Example TraceID for Flutter Gold:
/// ',CI=cirrus,Platform=linux,name=cupertino.activityIndicator.inprogress.1.0,source_type=flutter,'
String getTraceID(String testName) {
// If we are not in a CI environment, fallback on luci.
return '${platform.environment[_kTestBrowserKey] == null ? ',' : ',Browser=${platform.environment[_kTestBrowserKey]},'}'
'CI=${ci == ContinuousIntegrationEnvironment.none ? 'luci' : ci.toString().split('.').last},'
'Platform=${platform.operatingSystem},'
'name=$testName,'
'source_type=flutter,';
}
}
/// Used to make HttpRequests during testing.
class SkiaGoldHttpOverrides extends io.HttpOverrides {}
/// A digest returned from a request to the Flutter Gold dashboard.
class SkiaGoldDigest {
const SkiaGoldDigest({
required this.imageHash,
required this.paramSet,
required this.testName,
required this.status,
});
/// Create a digest from requested JSON.
factory SkiaGoldDigest.fromJson(Map<String, dynamic> json) {
return SkiaGoldDigest(
imageHash: json['digest'] as String,
paramSet:
Map<String, dynamic>.from(
json['refDiffs']['pos']['paramset'] as Map<String, dynamic>? ??
<String, List<String>>{
'Platform': <String>[],
'Browser' : <String>[],
}),
testName: json['test'] as String,
status: json['status'] as String,
);
}
/// Unique identifier for the image associated with the digest.
final String imageHash;
/// Parameter set for the given test, e.g. Platform : Windows.
final Map<String, dynamic> paramSet;
/// Test name associated with the digest, e.g. positive or un-triaged.
final String testName;
/// Status of the given digest, e.g. positive or un-triaged.
final String status;
/// Validates a given digest against the current testing conditions.
bool isValid(Platform platform, String name, String expectation) {
return imageHash == expectation
&& (paramSet['Platform'] as List<dynamic>/*!*/).contains(platform.operatingSystem)
&& (platform.environment[_kTestBrowserKey] == null
|| paramSet['Browser'] == platform.environment[_kTestBrowserKey])
&& testName == name
&& status == 'positive';
}
}