Refactor android_engine_test, make it easier to debug/deflake locally. (#161534)
The goal here is to have a great standalone `android_engine_test` suite that [replaces `scenario_app/android`](https://github.com/flutter/flutter/pull/160992). No test is _functionally_ changed in this PR, but overview of changes: - Finished renaming the suite `android_engine_tests` instead of `flutter_driver_android` - Added instructions and an environment variable for local generation of golden-files (`UPDATE_GOLDENS=1`) - Added explanations of the individual tests, where they live, and how to run them locally - Added a hybrid-composition (HC, not TLHC, which is already tested) test - Renamed "other_smiley" to "surface_texture_smiley" (and renamed the original to "surface_producer_smiley") - Removed unnecessary ".android" suffix (we will not run this on anything but Android) - Added a `tool/deflake.dart` to run a test suite 10x (or custom) times locally to try and determine flakiness After this PR, I'll add flags to let you control variants and name the screenshots accordingly, i.e.: - API v34 or v35 - OpenGLES or Vulkan (will require an `AndroidManifest.xml` edit during the test instrumentation)
This commit is contained in:
@@ -46,6 +46,12 @@ Future<void> enableSkiaGoldComparator({String? namePrefix}) async {
|
||||
'Set it to use Skia Gold.',
|
||||
);
|
||||
}
|
||||
if (namePrefix != null) {
|
||||
assert(
|
||||
!namePrefix.endsWith('.'),
|
||||
'The namePrefix automatically has a suffix of ".", so remove the last character from "$namePrefix".',
|
||||
);
|
||||
}
|
||||
final io.Directory tmpDir = io.Directory.systemTemp.createTempSync('android_driver_test');
|
||||
final bool isPresubmit = io.Platform.environment.containsKey(_kGoldctlPresubmitKey);
|
||||
io.stderr.writeln(
|
||||
@@ -126,12 +132,6 @@ final class _GoldenFileComparator extends GoldenFileComparator {
|
||||
'Golden files in the Flutter framework must end with the file extension '
|
||||
'.png.',
|
||||
);
|
||||
return Uri.parse(
|
||||
<String>[
|
||||
if (namePrefix != null) namePrefix!,
|
||||
baseDir.pathSegments[baseDir.pathSegments.length - 2],
|
||||
golden.toString(),
|
||||
].join('.'),
|
||||
);
|
||||
return Uri.parse(<String>[if (namePrefix != null) namePrefix!, golden.toString()].join('.'));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,16 @@ part of '../native_driver.dart';
|
||||
///
|
||||
/// When this is `true`, [matchesGoldenFile] will always report a successful
|
||||
/// match, because the bytes being tested implicitly become the new golden.
|
||||
bool autoUpdateGoldenFiles = false;
|
||||
///
|
||||
/// Defaults to `true` if the environment variable `UPDATE_GOLDENS` is either
|
||||
/// `true` or `1` (case insensitive).
|
||||
bool autoUpdateGoldenFiles = () {
|
||||
final String? updateGoldens = io.Platform.environment['UPDATE_GOLDENS'];
|
||||
return switch (updateGoldens?.toLowerCase()) {
|
||||
'1' || 'true' => true,
|
||||
_ => false,
|
||||
};
|
||||
}();
|
||||
|
||||
/// Compares pixels against those of a golden image file.
|
||||
///
|
||||
@@ -94,7 +103,17 @@ final class NaiveLocalFileComparator extends GoldenFileComparator {
|
||||
try {
|
||||
goldenBytes = await goldenFile.readAsBytes();
|
||||
} on io.PathNotFoundException {
|
||||
throw TestFailure('Golden file not found: ${goldenFile.path}');
|
||||
throw TestFailure(
|
||||
'Golden file not found: ${path.relative(goldenFile.path)}.\n'
|
||||
'\n'
|
||||
'For local development, you must establish a local baseline image before '
|
||||
'running tests, otherwise the test will always fail. Use UPDATE_GOLDENS=1 '
|
||||
'when running "flutter drive" to establish a baseline, and then subequent '
|
||||
'"flutter drive" instances will be tested against that (local) golden.\n'
|
||||
'\n'
|
||||
'See the documentation at dev/tools/android_engine_test/README.md for '
|
||||
'details.',
|
||||
);
|
||||
}
|
||||
|
||||
if (goldenBytes.length != imageBytes.length) {
|
||||
|
||||
Reference in New Issue
Block a user