diff --git a/dev/benchmarks/microbenchmarks/README.md b/dev/benchmarks/microbenchmarks/README.md index 157869ad10..a9fcf49d7d 100644 --- a/dev/benchmarks/microbenchmarks/README.md +++ b/dev/benchmarks/microbenchmarks/README.md @@ -1,22 +1,17 @@ # microbenchmarks -To run these benchmarks on a device, first run `flutter logs' in one -window to see the device logs, then, in a different window, run any of -these: +To run these benchmarks on a device, first run `flutter logs` in one +window to see the device logs, then, in a different window, run: ```sh -flutter run --release lib/gestures/velocity_tracker_bench.dart -flutter run --release lib/gestures/gesture_detector_bench.dart -flutter run --release lib/stocks/animation_bench.dart -flutter run --release lib/stocks/build_bench.dart -flutter run --release lib/stocks/layout_bench.dart +flutter run -d $DEVICE_ID --profile lib/benchmark_collection.dart ``` The results should be in the device logs. -### Avoid changing names of the benchmarks +## Avoid changing names of the benchmarks Each microbenchmark is identified by a name, for example, -"catmullrom_transform_iteration". Changing the name of an existing -microbenchmarks will effectively remove the old benchmark and create a new one, +"catmullrom_transform_iteration". Changing the name passed to `BenchmarkResultPrinter.addResult` +will effectively remove the old benchmark and create a new one, losing the historical data associated with the old benchmark in the process. diff --git a/dev/benchmarks/microbenchmarks/lib/benchmark_binding.dart b/dev/benchmarks/microbenchmarks/lib/benchmark_binding.dart new file mode 100644 index 0000000000..8ca2ba9d9e --- /dev/null +++ b/dev/benchmarks/microbenchmarks/lib/benchmark_binding.dart @@ -0,0 +1,23 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_test/flutter_test.dart'; + +class BenchmarkingBinding extends LiveTestWidgetsFlutterBinding { + BenchmarkingBinding(); + + final Stopwatch drawFrameWatch = Stopwatch(); + + @override + void handleBeginFrame(Duration? rawTimeStamp) { + drawFrameWatch.start(); + super.handleBeginFrame(rawTimeStamp); + } + + @override + void handleDrawFrame() { + super.handleDrawFrame(); + drawFrameWatch.stop(); + } +} diff --git a/dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart b/dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart new file mode 100644 index 0000000000..acb280c4f0 --- /dev/null +++ b/dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart @@ -0,0 +1,107 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:flutter/material.dart'; +import 'package:flutter/scheduler.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import 'benchmark_binding.dart'; +import 'foundation/all_elements_bench.dart' as all_elements_bench; +import 'foundation/change_notifier_bench.dart' as change_notifier_bench; +import 'foundation/clamp.dart' as clamp; +import 'foundation/decode_and_parse_asset_manifest.dart' + as decode_and_parse_asset_manifest; +import 'foundation/platform_asset_bundle.dart' as platform_asset_bundle; +import 'foundation/standard_message_codec_bench.dart' + as standard_message_codec_bench; +import 'foundation/standard_method_codec_bench.dart' + as standard_method_codec_bench; +import 'foundation/timeline_bench.dart' as timeline_bench; +import 'geometry/matrix_utils_transform_bench.dart' + as matrix_utils_transform_bench; +import 'geometry/rrect_contains_bench.dart' as rrect_contains_bench; +import 'gestures/gesture_detector_bench.dart' as gesture_detector_bench; +import 'gestures/velocity_tracker_bench.dart' as velocity_tracker_bench; +import 'language/compute_bench.dart' as compute_bench; +import 'language/sync_star_bench.dart' as sync_star_bench; +import 'language/sync_star_semantics_bench.dart' as sync_star_semantics_bench; +import 'layout/text_intrinsic_bench.dart' as text_intrinsic_bench; +import 'stocks/animation_bench.dart' as animation_bench; +import 'stocks/build_bench.dart' as build_bench; +import 'stocks/build_bench_profiled.dart' as build_bench_profiled; +import 'stocks/layout_bench.dart' as layout_bench; +import 'ui/image_bench.dart' as image_bench; + +typedef Benchmark = (String name, Future Function() value); + +Future main() async { + assert(false, + "Don't run benchmarks in debug mode! Use 'flutter run --release'."); + + // BenchmarkingBinding is used by animation_bench, providing a simple + // stopwatch interface over rendering. Lifting it here makes all + // benchmarks run together. + final BenchmarkingBinding binding = BenchmarkingBinding(); + final List benchmarks = [ + ('foundation/change_notifier_bench.dart', change_notifier_bench.execute), + ('foundation/clamp.dart', clamp.execute), + ('foundation/platform_asset_bundle.dart', platform_asset_bundle.execute), + ( + 'foundation/standard_message_codec_bench.dart', + standard_message_codec_bench.execute + ), + ( + 'foundation/standard_method_codec_bench.dart', + standard_method_codec_bench.execute + ), + ('foundation/timeline_bench.dart', timeline_bench.execute), + ( + 'foundation/decode_and_parse_asset_manifest.dart', + decode_and_parse_asset_manifest.execute + ), + ( + 'geometry/matrix_utils_transform_bench.dart', + matrix_utils_transform_bench.execute + ), + ('geometry/rrect_contains_bench.dart', rrect_contains_bench.execute), + ('gestures/gesture_detector_bench.dart', gesture_detector_bench.execute), + ('gestures/velocity_tracker_bench.dart', velocity_tracker_bench.execute), + ('language/compute_bench.dart', compute_bench.execute), + ('language/sync_star_bench.dart', sync_star_bench.execute), + ( + 'language/sync_star_semantics_bench.dart', + sync_star_semantics_bench.execute + ), + ('stocks/animation_bench.dart', () => animation_bench.execute(binding)), + ('stocks/build_bench.dart', build_bench.execute), + ('stocks/build_bench_profiled.dart', build_bench_profiled.execute), + ('stocks/layout_bench.dart', layout_bench.execute), + ('ui/image_bench.dart', image_bench.execute), + ('layout/text_intrinsic_bench.dart', text_intrinsic_bench.execute), + ( + 'foundation/all_elements_bench.dart', + () async { + binding.framePolicy = + LiveTestWidgetsFlutterBindingFramePolicy.fullyLive; + runApp(const SizedBox.shrink()); // ensure dispose + await SchedulerBinding.instance.endOfFrame; + all_elements_bench.execute(); + } + ), + ]; + + print('╡ ••• Running microbenchmarks ••• ╞'); + + for (final Benchmark mark in benchmarks) { + // Reset the frame policy to default - each test can set it on their own. + binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fadePointers; + print('╡ ••• Running ${mark.$1} ••• ╞'); + await mark.$2(); + } + + print('\n\n╡ ••• Done ••• ╞\n\n'); + exit(0); +} diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/all_elements_bench.dart b/dev/benchmarks/microbenchmarks/lib/foundation/all_elements_bench.dart index 23dd6d5169..fcfa798d69 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/all_elements_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/all_elements_bench.dart @@ -11,8 +11,7 @@ import '../common.dart'; const int _kNumIters = 10000; -Future main() async { - assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); +Future execute() async { runApp(MaterialApp( home: Scaffold( body: GridView.count( diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/change_notifier_bench.dart b/dev/benchmarks/microbenchmarks/lib/foundation/change_notifier_bench.dart index 0eedaef66b..3dcc8649bc 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/change_notifier_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/change_notifier_bench.dart @@ -10,7 +10,7 @@ const int _kNumIterations = 65536; const int _kNumWarmUp = 100; const int _kScale = 1000; -void main() { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); // In the following benchmarks, we won't remove the listeners when we don't diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/clamp.dart b/dev/benchmarks/microbenchmarks/lib/foundation/clamp.dart index 1a933193ab..3b1184da00 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/clamp.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/clamp.dart @@ -9,7 +9,7 @@ import '../common.dart'; const int _kBatchSize = 100000; const int _kNumIterations = 1000; -void main() { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/decode_and_parse_asset_manifest.dart b/dev/benchmarks/microbenchmarks/lib/foundation/decode_and_parse_asset_manifest.dart index 5082abcf2f..ccc3901a75 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/decode_and_parse_asset_manifest.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/decode_and_parse_asset_manifest.dart @@ -9,7 +9,7 @@ import '../common.dart'; const int _kNumIterations = 1000; -void main() async { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/platform_asset_bundle.dart b/dev/benchmarks/microbenchmarks/lib/foundation/platform_asset_bundle.dart index 4f2b172a10..e55d6183b4 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/platform_asset_bundle.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/platform_asset_bundle.dart @@ -10,7 +10,7 @@ import '../common.dart'; const int _kBatchSize = 100; const int _kNumIterations = 100; -void main() async { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/standard_message_codec_bench.dart b/dev/benchmarks/microbenchmarks/lib/foundation/standard_message_codec_bench.dart index 58aaa0f60f..daae70467e 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/standard_message_codec_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/standard_message_codec_bench.dart @@ -8,7 +8,7 @@ import '../common.dart'; const int _kNumIterations = 100000; -void main() { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/standard_method_codec_bench.dart b/dev/benchmarks/microbenchmarks/lib/foundation/standard_method_codec_bench.dart index 6ee3b64be4..a274befcf7 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/standard_method_codec_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/standard_method_codec_bench.dart @@ -8,7 +8,7 @@ import '../common.dart'; const int _kNumIterations = 100000; -void main() { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/timeline_bench.dart b/dev/benchmarks/microbenchmarks/lib/foundation/timeline_bench.dart index 7c9eb4a396..84b7fb8cbd 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/timeline_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/timeline_bench.dart @@ -8,7 +8,7 @@ import '../common.dart'; const int _kNumIterations = 10000; -void main() { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/geometry/matrix_utils_transform_bench.dart b/dev/benchmarks/microbenchmarks/lib/geometry/matrix_utils_transform_bench.dart index 35b9fecae9..b0096369d7 100644 --- a/dev/benchmarks/microbenchmarks/lib/geometry/matrix_utils_transform_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/geometry/matrix_utils_transform_bench.dart @@ -11,7 +11,7 @@ import '../common.dart'; const int _kNumIterations = 10000000; const int _kNumWarmUp = 100000; -void main() { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); print('MatrixUtils.transformRect and .transformPoint benchmark...'); diff --git a/dev/benchmarks/microbenchmarks/lib/geometry/rrect_contains_bench.dart b/dev/benchmarks/microbenchmarks/lib/geometry/rrect_contains_bench.dart index 4afd3c976e..de7429de64 100644 --- a/dev/benchmarks/microbenchmarks/lib/geometry/rrect_contains_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/geometry/rrect_contains_bench.dart @@ -8,7 +8,7 @@ import '../common.dart'; const int _kNumIters = 10000; -void main() { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final Stopwatch watch = Stopwatch(); print('RRect contains benchmark...'); diff --git a/dev/benchmarks/microbenchmarks/lib/gestures/gesture_detector_bench.dart b/dev/benchmarks/microbenchmarks/lib/gestures/gesture_detector_bench.dart index a5ca499555..6ee61e3afe 100644 --- a/dev/benchmarks/microbenchmarks/lib/gestures/gesture_detector_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/gestures/gesture_detector_bench.dart @@ -10,7 +10,7 @@ import 'apps/button_matrix_app.dart' as button_matrix; const int _kNumWarmUpIters = 20; const int _kNumIters = 300; -Future main() async { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final Stopwatch watch = Stopwatch(); print('GestureDetector semantics benchmark...'); diff --git a/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart b/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart index dcf4064525..996c306c25 100644 --- a/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart @@ -17,7 +17,7 @@ class TrackerBenchmark { final String name; } -Future main() async { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); final List benchmarks = [ diff --git a/dev/benchmarks/microbenchmarks/lib/language/compute_bench.dart b/dev/benchmarks/microbenchmarks/lib/language/compute_bench.dart index 36e8ef7527..59e08487b7 100644 --- a/dev/benchmarks/microbenchmarks/lib/language/compute_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/language/compute_bench.dart @@ -23,7 +23,7 @@ List test(int length) { (int index) => Data(index * index)); } -Future main() async { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); // Warm up lap diff --git a/dev/benchmarks/microbenchmarks/lib/language/sync_star_bench.dart b/dev/benchmarks/microbenchmarks/lib/language/sync_star_bench.dart index 2b34dcd6b4..41123e8e90 100644 --- a/dev/benchmarks/microbenchmarks/lib/language/sync_star_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/language/sync_star_bench.dart @@ -7,7 +7,7 @@ import '../common.dart'; const int _kNumIterations = 1000; const int _kNumWarmUp = 100; -void main() { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); // Warm up lap diff --git a/dev/benchmarks/microbenchmarks/lib/language/sync_star_semantics_bench.dart b/dev/benchmarks/microbenchmarks/lib/language/sync_star_semantics_bench.dart index ab8d098337..993d98b9b2 100644 --- a/dev/benchmarks/microbenchmarks/lib/language/sync_star_semantics_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/language/sync_star_semantics_bench.dart @@ -9,7 +9,7 @@ import '../common.dart'; const int _kNumIterations = 1000; const int _kNumWarmUp = 100; -void main() { +Future execute() async { final List words = 'Lorem Ipsum is simply dummy text of the printing and' " typesetting industry. Lorem Ipsum has been the industry's" ' standard dummy text ever since the 1500s, when an unknown' diff --git a/dev/benchmarks/microbenchmarks/lib/layout/text_intrinsic_bench.dart b/dev/benchmarks/microbenchmarks/lib/layout/text_intrinsic_bench.dart index 95da4a18fd..b43bea0b1a 100644 --- a/dev/benchmarks/microbenchmarks/lib/layout/text_intrinsic_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/layout/text_intrinsic_bench.dart @@ -20,7 +20,7 @@ final Widget intrinsicTextHeight = Directionality( ), ); -Future main() async { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); // We control the framePolicy below to prevent us from scheduling frames in diff --git a/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart b/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart index d9fcd4376b..ee5498019a 100644 --- a/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart @@ -7,35 +7,16 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:stocks/main.dart' as stocks; import 'package:stocks/stock_data.dart' as stock_data; +import '../benchmark_binding.dart'; import '../common.dart'; const Duration kBenchmarkTime = Duration(seconds: 15); -class BenchmarkingBinding extends LiveTestWidgetsFlutterBinding { - BenchmarkingBinding(this.stopwatch); - - final Stopwatch stopwatch; - - @override - void handleBeginFrame(Duration? rawTimeStamp) { - stopwatch.start(); - super.handleBeginFrame(rawTimeStamp); - } - - @override - void handleDrawFrame() { - super.handleDrawFrame(); - stopwatch.stop(); - } -} - -Future main() async { +Future execute(BenchmarkingBinding binding) async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); stock_data.StockData.actuallyFetchData = false; final Stopwatch wallClockWatch = Stopwatch(); - final Stopwatch cpuWatch = Stopwatch(); - BenchmarkingBinding(cpuWatch); int totalOpenFrameElapsedMicroseconds = 0; int totalOpenIterationCount = 0; @@ -52,27 +33,27 @@ Future main() async { bool drawerIsOpen = false; wallClockWatch.start(); while (wallClockWatch.elapsed < kBenchmarkTime) { - cpuWatch.reset(); + binding.drawFrameWatch.reset(); if (drawerIsOpen) { await tester.tapAt(const Offset(780.0, 250.0)); // Close drawer await tester.pump(); totalCloseIterationCount += 1; - totalCloseFrameElapsedMicroseconds += cpuWatch.elapsedMicroseconds; + totalCloseFrameElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds; } else { await tester.tapAt(const Offset(20.0, 50.0)); // Open drawer await tester.pump(); totalOpenIterationCount += 1; - totalOpenFrameElapsedMicroseconds += cpuWatch.elapsedMicroseconds; + totalOpenFrameElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds; } drawerIsOpen = !drawerIsOpen; // Time how long each frame takes - cpuWatch.reset(); + binding.drawFrameWatch.reset(); while (SchedulerBinding.instance.hasScheduledFrame) { await tester.pump(); totalSubsequentFramesIterationCount += 1; } - totalSubsequentFramesElapsedMicroseconds += cpuWatch.elapsedMicroseconds; + totalSubsequentFramesElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds; } }); diff --git a/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart b/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart index 6b868ba169..33eb5bd1d1 100644 --- a/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart @@ -55,7 +55,7 @@ Future> runBuildBenchmark() async { return values; } -Future main() async { +Future execute() async { final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); printer.addResultStatistics( description: 'Stock build', diff --git a/dev/benchmarks/microbenchmarks/lib/stocks/build_bench_profiled.dart b/dev/benchmarks/microbenchmarks/lib/stocks/build_bench_profiled.dart index 8f48ca7ee1..2774d461fe 100644 --- a/dev/benchmarks/microbenchmarks/lib/stocks/build_bench_profiled.dart +++ b/dev/benchmarks/microbenchmarks/lib/stocks/build_bench_profiled.dart @@ -8,7 +8,7 @@ import 'package:flutter_test/flutter_test.dart'; import '../common.dart'; import 'build_bench.dart'; -Future main() async { +Future execute() async { debugProfileBuildsEnabledUserWidgets = true; final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); printer.addResultStatistics( diff --git a/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart b/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart index 9d72b87aa3..e638ff7382 100644 --- a/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart @@ -12,7 +12,7 @@ import '../common.dart'; const Duration kBenchmarkTime = Duration(seconds: 15); -Future main() async { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); stock_data.StockData.actuallyFetchData = false; diff --git a/dev/benchmarks/microbenchmarks/lib/ui/image_bench.dart b/dev/benchmarks/microbenchmarks/lib/ui/image_bench.dart index 2b807acf3c..96a7e4ea5b 100644 --- a/dev/benchmarks/microbenchmarks/lib/ui/image_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/ui/image_bench.dart @@ -78,7 +78,7 @@ const List assets = [ // Measures the time it takes to load a fixed number of assets into an // immutable buffer to later be decoded by skia. -Future main() async { +Future execute() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final Stopwatch watch = Stopwatch(); diff --git a/dev/devicelab/lib/microbenchmarks.dart b/dev/devicelab/lib/microbenchmarks.dart index 451be27b1a..a218a35bc0 100644 --- a/dev/devicelab/lib/microbenchmarks.dart +++ b/dev/devicelab/lib/microbenchmarks.dart @@ -14,6 +14,8 @@ Future> readJsonResults(Process process) { const String jsonStart = '================ RESULTS ================'; const String jsonEnd = '================ FORMATTED =============='; const String jsonPrefix = ':::JSON:::'; + const String testComplete = '╡ ••• Done ••• ╞'; + bool jsonStarted = false; final StringBuffer jsonBuf = StringBuffer(); final Completer> completer = Completer>(); @@ -25,8 +27,9 @@ Future> readJsonResults(Process process) { stderr.writeln('[STDERR] $line'); }); + final List collectedJson = []; + bool processWasKilledIntentionally = false; - bool resultsHaveBeenParsed = false; final StreamSubscription stdoutSub = process.stdout .transform(const Utf8Decoder()) .transform(const LineSplitter()) @@ -38,48 +41,40 @@ Future> readJsonResults(Process process) { return; } - if (jsonStarted && line.contains(jsonEnd)) { - final String jsonOutput = jsonBuf.toString(); - - // If we end up here and have already parsed the results, it suggests that - // we have received output from another test because our `flutter run` - // process did not terminate correctly. - // https://github.com/flutter/flutter/issues/19096#issuecomment-402756549 - if (resultsHaveBeenParsed) { - throw 'Additional JSON was received after results has already been ' - 'processed. This suggests the `flutter run` process may have lived ' - 'past the end of our test and collected additional output from the ' - 'next test.\n\n' - 'The JSON below contains all collected output, including both from ' - 'the original test and what followed.\n\n' - '$jsonOutput'; - } - - jsonStarted = false; + if (line.contains(testComplete)) { processWasKilledIntentionally = true; - resultsHaveBeenParsed = true; // Sending a SIGINT/SIGTERM to the process here isn't reliable because [process] is // the shell (flutter is a shell script) and doesn't pass the signal on. // Sending a `q` is an instruction to quit using the console runner. // See https://github.com/flutter/flutter/issues/19208 process.stdin.write('q'); await process.stdin.flush(); - // Give the process a couple of seconds to exit and run shutdown hooks // before sending kill signal. // TODO(fujino): https://github.com/flutter/flutter/issues/134566 await Future.delayed(const Duration(seconds: 2)); - // Also send a kill signal in case the `q` above didn't work. process.kill(ProcessSignal.sigint); try { - completer.complete(Map.from(json.decode(jsonOutput) as Map)); + final Map results = + Map.from({ + for (final String data in collectedJson) + ...json.decode(data) as Map + }); + completer.complete(results); } catch (ex) { - completer.completeError('Decoding JSON failed ($ex). JSON string was: $jsonOutput'); + completer.completeError( + 'Decoding JSON failed ($ex). JSON strings where: $collectedJson'); } return; } + if (jsonStarted && line.contains(jsonEnd)) { + collectedJson.add(jsonBuf.toString().trim()); + jsonBuf.clear(); + jsonStarted = false; + } + if (jsonStarted && line.contains(jsonPrefix)) { jsonBuf.writeln(line.substring(line.indexOf(jsonPrefix) + jsonPrefix.length)); } diff --git a/dev/devicelab/lib/tasks/microbenchmarks.dart b/dev/devicelab/lib/tasks/microbenchmarks.dart index aa38ab9a57..b5e17f3c22 100644 --- a/dev/devicelab/lib/tasks/microbenchmarks.dart +++ b/dev/devicelab/lib/tasks/microbenchmarks.dart @@ -24,11 +24,27 @@ TaskFunction createMicrobenchmarkTask({ await device.unlock(); await device.clearLogs(); + final Directory appDir = + dir(path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks')); + + // Hard-uninstall any prior apps. + await inDirectory(appDir, () async { + section('Uninstall previous microbenchmarks app'); + await flutter( + 'install', + options: [ + '-v', + '--uninstall-only', + '-d', + device.deviceId, + ], + ); + }); + Future> runMicrobench(String benchmarkPath) async { Future> run() async { print('Running $benchmarkPath'); - final Directory appDir = dir( - path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks')); + final Process flutterProcess = await inDirectory(appDir, () async { final List options = [ '-v', @@ -54,27 +70,7 @@ TaskFunction createMicrobenchmarkTask({ } final Map allResults = { - ...await runMicrobench('lib/foundation/all_elements_bench.dart'), - ...await runMicrobench('lib/foundation/change_notifier_bench.dart'), - ...await runMicrobench('lib/foundation/clamp.dart'), - ...await runMicrobench('lib/foundation/platform_asset_bundle.dart'), - ...await runMicrobench('lib/foundation/standard_message_codec_bench.dart'), - ...await runMicrobench('lib/foundation/standard_method_codec_bench.dart'), - ...await runMicrobench('lib/foundation/timeline_bench.dart'), - ...await runMicrobench('lib/foundation/decode_and_parse_asset_manifest.dart'), - ...await runMicrobench('lib/geometry/matrix_utils_transform_bench.dart'), - ...await runMicrobench('lib/geometry/rrect_contains_bench.dart'), - ...await runMicrobench('lib/gestures/gesture_detector_bench.dart'), - ...await runMicrobench('lib/gestures/velocity_tracker_bench.dart'), - ...await runMicrobench('lib/language/compute_bench.dart'), - ...await runMicrobench('lib/language/sync_star_bench.dart'), - ...await runMicrobench('lib/language/sync_star_semantics_bench.dart'), - ...await runMicrobench('lib/stocks/animation_bench.dart'), - ...await runMicrobench('lib/stocks/build_bench_profiled.dart'), - ...await runMicrobench('lib/stocks/build_bench.dart'), - ...await runMicrobench('lib/stocks/layout_bench.dart'), - ...await runMicrobench('lib/ui/image_bench.dart'), - ...await runMicrobench('lib/layout/text_intrinsic_bench.dart'), + ...await runMicrobench('lib/benchmark_collection.dart'), }; return TaskResult.success(allResults,