zan1456/flutter
1
0
Fork 0
forked from firka/flutter
Code Pull Requests Activity

Run all microbenchmarks (part trois) (#154446)

Browse Source 
**Three things**

Re-lands #154374

New: fix `platform_channels_benchmarks` to print the "done" key. Updated notes for the microbenchmark parser. There are no other users of `microbenchmarks.readJsonResults`.

Re-Re-land: Uninstall microbenchmarks before running them.
Flakes in https://github.com/flutter/flutter/issues/153828 stem from adb saying the app isn't installed, but then failing to install wtih -r. Several other tests uninstall the app before trying to run it. Previous fix called uninstall between tests, but iOS takes 12 to 13 seconds to perform uninstall / install, which timed out the test. Just uninstall the one time since we only care about any lingering apps with different keys.
Potential solution https://github.com/flutter/flutter/issues/153828

Re-land Make things go fast
Instead of installing 21 different compilations of the same app to get results; compile and run them together. Locally on Mac+iOS, this should takes ~3 minutes instead of ~15 minutes.
This commit is contained in:
John McDole
2024-08-30 13:55:16 -07:00
committed by GitHub
parent 8c1a93508b
commit 7aace1c5f9
27 changed files with 206 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
dev/devicelab/lib/microbenchmarks.dart
View File

@@ -9,11 +9,16 @@ import 'dart:io';
/// Reads through the print commands from [process] waiting for the magic phase
/// that contains microbenchmarks results as defined in
/// `dev/benchmarks/microbenchmarks/lib/common.dart`.
///
/// If you are using this outside of microbenchmarks, ensure you print a single
/// line with `╡ ••• Done ••• ╞` to signal the end of collection.
Future<Map<String, double>> readJsonResults(Process process) {
// IMPORTANT: keep these values in sync with dev/benchmarks/microbenchmarks/lib/common.dart
const String jsonStart = '================ RESULTS ================';
const String jsonEnd = '================ FORMATTED ==============';
const String jsonPrefix = ':::JSON:::';
const String testComplete = '╡ ••• Done ••• ╞';
bool jsonStarted = false;
final StringBuffer jsonBuf = StringBuffer();
final Completer<Map<String, double>> completer = Completer<Map<String, double>>();
@@ -25,8 +30,9 @@ Future<Map<String, double>> readJsonResults(Process process) {
stderr.writeln('[STDERR] $line');
});
final List<String> collectedJson = <String>[];
bool processWasKilledIntentionally = false;
bool resultsHaveBeenParsed = false;
final StreamSubscription<String> stdoutSub = process.stdout
.transform<String>(const Utf8Decoder())
.transform<String>(const LineSplitter())
@@ -38,48 +44,40 @@ Future<Map<String, double>> readJsonResults(Process process) {
return;
}
if (jsonStarted && line.contains(jsonEnd)) {
final String jsonOutput = jsonBuf.toString();
// If we end up here and have already parsed the results, it suggests that
// we have received output from another test because our `flutter run`
// process did not terminate correctly.
// https://github.com/flutter/flutter/issues/19096#issuecomment-402756549
if (resultsHaveBeenParsed) {
throw 'Additional JSON was received after results has already been '
'processed. This suggests the `flutter run` process may have lived '
'past the end of our test and collected additional output from the '
'next test.\n\n'
'The JSON below contains all collected output, including both from '
'the original test and what followed.\n\n'
'$jsonOutput';
}
jsonStarted = false;
if (line.contains(testComplete)) {
processWasKilledIntentionally = true;
resultsHaveBeenParsed = true;
// Sending a SIGINT/SIGTERM to the process here isn't reliable because [process] is
// the shell (flutter is a shell script) and doesn't pass the signal on.
// Sending a `q` is an instruction to quit using the console runner.
// See https://github.com/flutter/flutter/issues/19208
process.stdin.write('q');
await process.stdin.flush();
// Give the process a couple of seconds to exit and run shutdown hooks
// before sending kill signal.
// TODO(fujino): https://github.com/flutter/flutter/issues/134566
await Future<void>.delayed(const Duration(seconds: 2));
// Also send a kill signal in case the `q` above didn't work.
process.kill(ProcessSignal.sigint);
try {
completer.complete(Map<String, double>.from(json.decode(jsonOutput) as Map<String, dynamic>));
final Map<String, double> results =
Map<String, double>.from(<String, dynamic>{
for (final String data in collectedJson)
...json.decode(data) as Map<String, dynamic>
});
completer.complete(results);
} catch (ex) {
completer.completeError('Decoding JSON failed ($ex). JSON string was: $jsonOutput');
completer.completeError(
'Decoding JSON failed ($ex). JSON strings where: $collectedJson');
}
return;
}
if (jsonStarted && line.contains(jsonEnd)) {
collectedJson.add(jsonBuf.toString().trim());
jsonBuf.clear();
jsonStarted = false;
}
if (jsonStarted && line.contains(jsonPrefix)) {
jsonBuf.writeln(line.substring(line.indexOf(jsonPrefix) + jsonPrefix.length));
}

42
dev/devicelab/lib/tasks/microbenchmarks.dart
View File

@@ -24,11 +24,27 @@ TaskFunction createMicrobenchmarkTask({
await device.unlock();
await device.clearLogs();
final Directory appDir =
dir(path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks'));
// Hard-uninstall any prior apps.
await inDirectory(appDir, () async {
section('Uninstall previous microbenchmarks app');
await flutter(
'install',
options: <String>[
'-v',
'--uninstall-only',
'-d',
device.deviceId,
],
);
});
Future<Map<String, double>> runMicrobench(String benchmarkPath) async {
Future<Map<String, double>> run() async {
print('Running $benchmarkPath');
final Directory appDir = dir(
path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks'));
final Process flutterProcess = await inDirectory(appDir, () async {
final List<String> options = <String>[
'-v',
@@ -54,27 +70,7 @@ TaskFunction createMicrobenchmarkTask({
}
final Map<String, double> allResults = <String, double>{
...await runMicrobench('lib/foundation/all_elements_bench.dart'),
...await runMicrobench('lib/foundation/change_notifier_bench.dart'),
...await runMicrobench('lib/foundation/clamp.dart'),
...await runMicrobench('lib/foundation/platform_asset_bundle.dart'),
...await runMicrobench('lib/foundation/standard_message_codec_bench.dart'),
...await runMicrobench('lib/foundation/standard_method_codec_bench.dart'),
...await runMicrobench('lib/foundation/timeline_bench.dart'),
...await runMicrobench('lib/foundation/decode_and_parse_asset_manifest.dart'),
...await runMicrobench('lib/geometry/matrix_utils_transform_bench.dart'),
...await runMicrobench('lib/geometry/rrect_contains_bench.dart'),
...await runMicrobench('lib/gestures/gesture_detector_bench.dart'),
...await runMicrobench('lib/gestures/velocity_tracker_bench.dart'),
...await runMicrobench('lib/language/compute_bench.dart'),
...await runMicrobench('lib/language/sync_star_bench.dart'),
...await runMicrobench('lib/language/sync_star_semantics_bench.dart'),
...await runMicrobench('lib/stocks/animation_bench.dart'),
...await runMicrobench('lib/stocks/build_bench_profiled.dart'),
...await runMicrobench('lib/stocks/build_bench.dart'),
...await runMicrobench('lib/stocks/layout_bench.dart'),
...await runMicrobench('lib/ui/image_bench.dart'),
...await runMicrobench('lib/layout/text_intrinsic_bench.dart'),
...await runMicrobench('lib/benchmark_collection.dart'),
};
return TaskResult.success(allResults,