Refactor android_engine_test, make it easier to debug/deflake locally. (#161534)

The goal here is to have a great standalone `android_engine_test` suite
that [replaces
`scenario_app/android`](https://github.com/flutter/flutter/pull/160992).

No test is _functionally_ changed in this PR, but overview of changes:
- Finished renaming the suite `android_engine_tests` instead of
`flutter_driver_android`
- Added instructions and an environment variable for local generation of
golden-files (`UPDATE_GOLDENS=1`)
- Added explanations of the individual tests, where they live, and how
to run them locally
- Added a hybrid-composition (HC, not TLHC, which is already tested)
test
- Renamed "other_smiley" to "surface_texture_smiley" (and renamed the
original to "surface_producer_smiley")
- Removed unnecessary ".android" suffix (we will not run this on
anything but Android)
- Added a `tool/deflake.dart` to run a test suite 10x (or custom) times
locally to try and determine flakiness

After this PR, I'll add flags to let you control variants and name the
screenshots accordingly, i.e.:
- API v34 or v35
- OpenGLES or Vulkan (will require an `AndroidManifest.xml` edit during
the test instrumentation)
This commit is contained in:
Matan Lurey
2025-01-14 17:05:31 -08:00
committed by GitHub
parent b515f829af
commit e517ae3457
17 changed files with 554 additions and 74 deletions

View File

@@ -46,6 +46,12 @@ Future<void> enableSkiaGoldComparator({String? namePrefix}) async {
'Set it to use Skia Gold.',
);
}
if (namePrefix != null) {
assert(
!namePrefix.endsWith('.'),
'The namePrefix automatically has a suffix of ".", so remove the last character from "$namePrefix".',
);
}
final io.Directory tmpDir = io.Directory.systemTemp.createTempSync('android_driver_test');
final bool isPresubmit = io.Platform.environment.containsKey(_kGoldctlPresubmitKey);
io.stderr.writeln(
@@ -126,12 +132,6 @@ final class _GoldenFileComparator extends GoldenFileComparator {
'Golden files in the Flutter framework must end with the file extension '
'.png.',
);
return Uri.parse(
<String>[
if (namePrefix != null) namePrefix!,
baseDir.pathSegments[baseDir.pathSegments.length - 2],
golden.toString(),
].join('.'),
);
return Uri.parse(<String>[if (namePrefix != null) namePrefix!, golden.toString()].join('.'));
}
}

View File

@@ -16,7 +16,16 @@ part of '../native_driver.dart';
///
/// When this is `true`, [matchesGoldenFile] will always report a successful
/// match, because the bytes being tested implicitly become the new golden.
bool autoUpdateGoldenFiles = false;
///
/// Defaults to `true` if the environment variable `UPDATE_GOLDENS` is either
/// `true` or `1` (case insensitive).
bool autoUpdateGoldenFiles = () {
final String? updateGoldens = io.Platform.environment['UPDATE_GOLDENS'];
return switch (updateGoldens?.toLowerCase()) {
'1' || 'true' => true,
_ => false,
};
}();
/// Compares pixels against those of a golden image file.
///
@@ -94,7 +103,17 @@ final class NaiveLocalFileComparator extends GoldenFileComparator {
try {
goldenBytes = await goldenFile.readAsBytes();
} on io.PathNotFoundException {
throw TestFailure('Golden file not found: ${goldenFile.path}');
throw TestFailure(
'Golden file not found: ${path.relative(goldenFile.path)}.\n'
'\n'
'For local development, you must establish a local baseline image before '
'running tests, otherwise the test will always fail. Use UPDATE_GOLDENS=1 '
'when running "flutter drive" to establish a baseline, and then subequent '
'"flutter drive" instances will be tested against that (local) golden.\n'
'\n'
'See the documentation at dev/tools/android_engine_test/README.md for '
'details.',
);
}
if (goldenBytes.length != imageBytes.length) {