firka/flutter
3
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Try golden-testing on a Mokey (bringup: true), retry on an emulator (#163029)

Browse Source 
Towards https://github.com/flutter/flutter/issues/162362 (see if retries
help VD on an emulator).
Towards https://github.com/flutter/flutter/issues/163025 (see if we can
golden-test on a device).

May the odds be in our favor.
This commit is contained in:
Matan Lurey
2025-02-10 17:46:24 -08:00
committed by GitHub
parent af3c91045b
commit 7afc8557a4
8 changed files with 115 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
.ci.yaml
View File

@@ -1548,6 +1548,20 @@ targets:
{"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"}
]
- name: Linux_mokey android_engine_vulkan_tests
recipe: flutter/flutter_drone
# TODO(matanlurey): https://github.com/flutter/flutter/issues/163025.
bringup: true
timeout: 60
properties:
shard: android_engine_vulkan_tests
tags: >
["framework", "hostonly", "shard", "linux"]
dependencies: >-
[
{"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"}
]
- name: Linux_android_emu android_engine_opengles_tests
recipe: flutter/flutter_drone
timeout: 60

8
dev/integration_tests/android_engine_test/android/app/src/main/kotlin/com/example/native_driver_test/extensions/NativeDriverSupportPlugin.kt
View File

@@ -49,6 +49,14 @@ class NativeDriverSupportPlugin :
val versionMap = mapOf("version" to Build.VERSION.SDK_INT)
result.success(versionMap)
}
"is_emulator" -> {
val isEmulator =
when {
Build.MODEL.contains("gphone") -> true
else -> false
}
result.success(mapOf("emulator" to isEmulator))
}
"ping" -> {
result.success(null)
}

56
dev/integration_tests/android_engine_test/test_driver/_unstable_gold_retry.dart Normal file
View File

@@ -0,0 +1,56 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
/// @docImport 'package:android_driver_extensions/skia_gold.dart';
library;
// Similar to `flutter_test`, we ignore the implementation import.
// ignore: implementation_imports
import 'package:android_driver_extensions/native_driver.dart';
import 'package:matcher/src/expect/async_matcher.dart';
import 'package:matcher/src/interfaces.dart';
/// Invokes [matchesGoldenFile] with optional [retries] if a comparison fails.
AsyncMatcher matchesGoldenFileWithRetries(Object key, {int? version, int retries = 2}) {
final AsyncMatcher delegate = matchesGoldenFile(key, version: version);
if (retries == 0) {
return delegate;
}
return _AsyncMatcherWithRetries(delegate, retries: retries);
}
final class _AsyncMatcherWithRetries extends AsyncMatcher {
_AsyncMatcherWithRetries(this._delegate, {required int retries}) : _retries = retries {
if (retries < 1) {
throw RangeError.value(retries, 'retries', 'Must be at least 1');
}
}
final AsyncMatcher _delegate;
int _retries;
@override
Description describe(Description description) {
description = _delegate.describe(description);
description.add('Retries remaining: $_retries');
return description;
}
@override
Future<String?> matchAsync(Object? item) async {
while (true) {
final Object? error = await _delegate.matchAsync(item);
if (error == null) {
return null;
}
print('Failed: $error');
if (--_retries == 0) {
return 'Retries exceeded. Giving up.';
} else {
print('Retrying... $_retries retries left.');
}
assert(_retries >= 0, 'Unreachable');
}
}
}

21
dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart
View File

@@ -8,6 +8,7 @@ import 'package:flutter_driver/flutter_driver.dart';
import 'package:test/test.dart';
import '../_luci_skia_gold_prelude.dart';
import '../_unstable_gold_retry.dart';
/// For local debugging, a (local) golden-file is required as a baseline:
///
@@ -49,22 +50,10 @@ void main() async {
// See:
// - Vulkan: https://github.com/flutter/flutter/issues/162362
// - OpenGLES: https://github.com/flutter/flutter/issues/162363
int retriesLeft = 2;
do {
try {
await expectLater(
nativeDriver.screenshot(),
matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.png'),
);
break;
} on TestFailure catch (e) {
if (retriesLeft == 0) {
rethrow;
}
print('Caught: $e. Retrying...');
retriesLeft--;
}
} while (retriesLeft > 0);
await expectLater(
nativeDriver.screenshot(),
matchesGoldenFileWithRetries('$goldenPrefix.blue_orange_gradient_portrait.png'),
);
}, timeout: Timeout.none);
test('should rotate landscape and screenshot the gradient', () async {

21
dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart
View File

@@ -8,6 +8,7 @@ import 'package:flutter_driver/flutter_driver.dart';
import 'package:test/test.dart';
import '../_luci_skia_gold_prelude.dart';
import '../_unstable_gold_retry.dart';
/// For local debugging, a (local) golden-file is required as a baseline:
///
@@ -28,6 +29,9 @@ void main() async {
late final FlutterDriver flutterDriver;
late final NativeDriver nativeDriver;
late final bool isEmulator;
late final bool isVulkan;
setUpAll(() async {
if (isLuci) {
await enableSkiaGoldComparator(namePrefix: 'android_engine_test$goldenVariant');
@@ -42,6 +46,13 @@ void main() async {
if (await nativeDriver.sdkVersion case final int version when version < 23) {
fail('Requires SDK >= 23, got $version');
}
// TODO(matanlurey): https://github.com/flutter/flutter/issues/162362#issuecomment-2649555821.
isEmulator = await nativeDriver.isEmulator;
isVulkan = goldenVariant.contains('vulkan');
if (isEmulator && isVulkan) {
print('Detected running on a vulkan emulator. Will retry certain failures');
}
});
tearDownAll(() async {
@@ -60,13 +71,19 @@ void main() async {
await nativeDriver.rotateToLandscape();
await expectLater(
nativeDriver.screenshot(),
matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.png'),
matchesGoldenFileWithRetries(
'$goldenPrefix.blue_orange_gradient_landscape_rotated.png',
retries: isEmulator && isVulkan ? 2 : 0,
),
);
await nativeDriver.rotateResetDefault();
await expectLater(
nativeDriver.screenshot(),
matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.png'),
matchesGoldenFileWithRetries(
'$goldenPrefix.blue_orange_gradient_portait_rotated_back.png',
retries: isEmulator && isVulkan ? 2 : 0,
),
);
}, timeout: Timeout.none);
}

6
dev/tools/android_driver_extensions/lib/src/backend/android/driver.dart
View File

@@ -77,6 +77,12 @@ final class AndroidNativeDriver implements NativeDriver {
return result['version']! as int;
}
@override
Future<bool> get isEmulator async {
final Map<String, Object?> result = await _driver.sendCommand(NativeCommand.getIsEmulator);
return result['emulator']! as bool;
}
/// Waits for 2 seconds before completing.
///
/// There is no perfect way, outside of polling, to know when the device is

3
dev/tools/android_driver_extensions/lib/src/common.dart
View File

@@ -30,6 +30,9 @@ final class NativeCommand extends Command {
/// Gets the SDK version code.
static const NativeCommand getSdkVersion = NativeCommand('sdk_version');
/// Gets whether the device is an emulator.
static const NativeCommand getIsEmulator = NativeCommand('is_emulator');
/// The method to call on the plugin.
final String method;

5
dev/tools/android_driver_extensions/lib/src/driver.dart
View File

@@ -45,9 +45,12 @@ abstract interface class NativeDriver {
/// ```
Future<Duration> ping();
/// Returns the SDK version.
/// The SDK version.
Future<int> get sdkVersion;
/// Whether the device is an emulator.
Future<bool> get isEmulator;
/// Take a screenshot using a platform-specific mechanism.
///
/// The image is returned as an opaque handle that can be used to retrieve