diff --git a/.ci.yaml b/.ci.yaml index 69aaf97e85..51eb0f667f 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -1548,6 +1548,20 @@ targets: {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} ] + - name: Linux_mokey android_engine_vulkan_tests + recipe: flutter/flutter_drone + # TODO(matanlurey): https://github.com/flutter/flutter/issues/163025. + bringup: true + timeout: 60 + properties: + shard: android_engine_vulkan_tests + tags: > + ["framework", "hostonly", "shard", "linux"] + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] + - name: Linux_android_emu android_engine_opengles_tests recipe: flutter/flutter_drone timeout: 60 diff --git a/dev/integration_tests/android_engine_test/android/app/src/main/kotlin/com/example/native_driver_test/extensions/NativeDriverSupportPlugin.kt b/dev/integration_tests/android_engine_test/android/app/src/main/kotlin/com/example/native_driver_test/extensions/NativeDriverSupportPlugin.kt index acb9fbd136..576c60fb92 100644 --- a/dev/integration_tests/android_engine_test/android/app/src/main/kotlin/com/example/native_driver_test/extensions/NativeDriverSupportPlugin.kt +++ b/dev/integration_tests/android_engine_test/android/app/src/main/kotlin/com/example/native_driver_test/extensions/NativeDriverSupportPlugin.kt @@ -49,6 +49,14 @@ class NativeDriverSupportPlugin : val versionMap = mapOf("version" to Build.VERSION.SDK_INT) result.success(versionMap) } + "is_emulator" -> { + val isEmulator = + when { + Build.MODEL.contains("gphone") -> true + else -> false + } + result.success(mapOf("emulator" to isEmulator)) + } "ping" -> { result.success(null) } diff --git a/dev/integration_tests/android_engine_test/test_driver/_unstable_gold_retry.dart b/dev/integration_tests/android_engine_test/test_driver/_unstable_gold_retry.dart new file mode 100644 index 0000000000..462acd7b2d --- /dev/null +++ b/dev/integration_tests/android_engine_test/test_driver/_unstable_gold_retry.dart @@ -0,0 +1,56 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// @docImport 'package:android_driver_extensions/skia_gold.dart'; +library; + +// Similar to `flutter_test`, we ignore the implementation import. +// ignore: implementation_imports +import 'package:android_driver_extensions/native_driver.dart'; +import 'package:matcher/src/expect/async_matcher.dart'; +import 'package:matcher/src/interfaces.dart'; + +/// Invokes [matchesGoldenFile] with optional [retries] if a comparison fails. +AsyncMatcher matchesGoldenFileWithRetries(Object key, {int? version, int retries = 2}) { + final AsyncMatcher delegate = matchesGoldenFile(key, version: version); + if (retries == 0) { + return delegate; + } + return _AsyncMatcherWithRetries(delegate, retries: retries); +} + +final class _AsyncMatcherWithRetries extends AsyncMatcher { + _AsyncMatcherWithRetries(this._delegate, {required int retries}) : _retries = retries { + if (retries < 1) { + throw RangeError.value(retries, 'retries', 'Must be at least 1'); + } + } + + final AsyncMatcher _delegate; + int _retries; + + @override + Description describe(Description description) { + description = _delegate.describe(description); + description.add('Retries remaining: $_retries'); + return description; + } + + @override + Future matchAsync(Object? item) async { + while (true) { + final Object? error = await _delegate.matchAsync(item); + if (error == null) { + return null; + } + print('Failed: $error'); + if (--_retries == 0) { + return 'Retries exceeded. Giving up.'; + } else { + print('Retrying... $_retries retries left.'); + } + assert(_retries >= 0, 'Unreachable'); + } + } +} diff --git a/dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart index 14acb3e86a..9b04fa99b2 100644 --- a/dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart @@ -8,6 +8,7 @@ import 'package:flutter_driver/flutter_driver.dart'; import 'package:test/test.dart'; import '../_luci_skia_gold_prelude.dart'; +import '../_unstable_gold_retry.dart'; /// For local debugging, a (local) golden-file is required as a baseline: /// @@ -49,22 +50,10 @@ void main() async { // See: // - Vulkan: https://github.com/flutter/flutter/issues/162362 // - OpenGLES: https://github.com/flutter/flutter/issues/162363 - int retriesLeft = 2; - do { - try { - await expectLater( - nativeDriver.screenshot(), - matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.png'), - ); - break; - } on TestFailure catch (e) { - if (retriesLeft == 0) { - rethrow; - } - print('Caught: $e. Retrying...'); - retriesLeft--; - } - } while (retriesLeft > 0); + await expectLater( + nativeDriver.screenshot(), + matchesGoldenFileWithRetries('$goldenPrefix.blue_orange_gradient_portrait.png'), + ); }, timeout: Timeout.none); test('should rotate landscape and screenshot the gradient', () async { diff --git a/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart index a987990a54..510271c3b5 100644 --- a/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart @@ -8,6 +8,7 @@ import 'package:flutter_driver/flutter_driver.dart'; import 'package:test/test.dart'; import '../_luci_skia_gold_prelude.dart'; +import '../_unstable_gold_retry.dart'; /// For local debugging, a (local) golden-file is required as a baseline: /// @@ -28,6 +29,9 @@ void main() async { late final FlutterDriver flutterDriver; late final NativeDriver nativeDriver; + late final bool isEmulator; + late final bool isVulkan; + setUpAll(() async { if (isLuci) { await enableSkiaGoldComparator(namePrefix: 'android_engine_test$goldenVariant'); @@ -42,6 +46,13 @@ void main() async { if (await nativeDriver.sdkVersion case final int version when version < 23) { fail('Requires SDK >= 23, got $version'); } + + // TODO(matanlurey): https://github.com/flutter/flutter/issues/162362#issuecomment-2649555821. + isEmulator = await nativeDriver.isEmulator; + isVulkan = goldenVariant.contains('vulkan'); + if (isEmulator && isVulkan) { + print('Detected running on a vulkan emulator. Will retry certain failures'); + } }); tearDownAll(() async { @@ -60,13 +71,19 @@ void main() async { await nativeDriver.rotateToLandscape(); await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.png'), + matchesGoldenFileWithRetries( + '$goldenPrefix.blue_orange_gradient_landscape_rotated.png', + retries: isEmulator && isVulkan ? 2 : 0, + ), ); await nativeDriver.rotateResetDefault(); await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.png'), + matchesGoldenFileWithRetries( + '$goldenPrefix.blue_orange_gradient_portait_rotated_back.png', + retries: isEmulator && isVulkan ? 2 : 0, + ), ); }, timeout: Timeout.none); } diff --git a/dev/tools/android_driver_extensions/lib/src/backend/android/driver.dart b/dev/tools/android_driver_extensions/lib/src/backend/android/driver.dart index 7b8fec8bf8..5c0ba8fddf 100644 --- a/dev/tools/android_driver_extensions/lib/src/backend/android/driver.dart +++ b/dev/tools/android_driver_extensions/lib/src/backend/android/driver.dart @@ -77,6 +77,12 @@ final class AndroidNativeDriver implements NativeDriver { return result['version']! as int; } + @override + Future get isEmulator async { + final Map result = await _driver.sendCommand(NativeCommand.getIsEmulator); + return result['emulator']! as bool; + } + /// Waits for 2 seconds before completing. /// /// There is no perfect way, outside of polling, to know when the device is diff --git a/dev/tools/android_driver_extensions/lib/src/common.dart b/dev/tools/android_driver_extensions/lib/src/common.dart index 8d8e0f2572..937ad303b1 100644 --- a/dev/tools/android_driver_extensions/lib/src/common.dart +++ b/dev/tools/android_driver_extensions/lib/src/common.dart @@ -30,6 +30,9 @@ final class NativeCommand extends Command { /// Gets the SDK version code. static const NativeCommand getSdkVersion = NativeCommand('sdk_version'); + /// Gets whether the device is an emulator. + static const NativeCommand getIsEmulator = NativeCommand('is_emulator'); + /// The method to call on the plugin. final String method; diff --git a/dev/tools/android_driver_extensions/lib/src/driver.dart b/dev/tools/android_driver_extensions/lib/src/driver.dart index f6af26e2b8..f11fcaaf56 100644 --- a/dev/tools/android_driver_extensions/lib/src/driver.dart +++ b/dev/tools/android_driver_extensions/lib/src/driver.dart @@ -45,9 +45,12 @@ abstract interface class NativeDriver { /// ``` Future ping(); - /// Returns the SDK version. + /// The SDK version. Future get sdkVersion; + /// Whether the device is an emulator. + Future get isEmulator; + /// Take a screenshot using a platform-specific mechanism. /// /// The image is returned as an opaque handle that can be used to retrieve