From 8c264775bf7db1297c2a65e36f648145d81092a0 Mon Sep 17 00:00:00 2001 From: "auto-submit[bot]" <98614782+auto-submit[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 19:44:19 +0000 Subject: [PATCH] Reverts "Run all microbenchmarks (#154374)" (#154440) Reverts: flutter/flutter#154374 Initiated by: jtmcdole Reason for reverting: A different benchmark was using the microbenchmark parser and timing out. Original PR Author: jtmcdole Reviewed By: {zanderso} This change reverts the following previous change: Two things: **Re-land**: Uninstall microbenchmarks before running them. Flakes in #153828 stem from adb saying the app isn't installed, but then failing to install wtih `-r`. Several other tests uninstall the app before trying to run it. Previous fix called uninstall between tests, but iOS takes 12 to 13 seconds to perform uninstall / install, which timed out the test. Just uninstall the one time since we only care about any lingering apps with different keys. Potential solution #153828 **Make things go fast** Instead of installing 21 different compilations of the same app to get results; compile and run them together. Locally on Mac+iOS, this should takes ~3 minutes instead of ~15 minutes. --- dev/benchmarks/microbenchmarks/README.md | 17 ++- .../lib/benchmark_binding.dart | 23 ---- .../lib/benchmark_collection.dart | 107 ------------------ .../lib/foundation/all_elements_bench.dart | 3 +- .../lib/foundation/change_notifier_bench.dart | 2 +- .../microbenchmarks/lib/foundation/clamp.dart | 2 +- .../decode_and_parse_asset_manifest.dart | 2 +- .../lib/foundation/platform_asset_bundle.dart | 2 +- .../standard_message_codec_bench.dart | 2 +- .../standard_method_codec_bench.dart | 2 +- .../lib/foundation/timeline_bench.dart | 2 +- .../matrix_utils_transform_bench.dart | 2 +- .../lib/geometry/rrect_contains_bench.dart | 2 +- .../lib/gestures/gesture_detector_bench.dart | 2 +- .../lib/gestures/velocity_tracker_bench.dart | 2 +- .../lib/language/compute_bench.dart | 2 +- .../lib/language/sync_star_bench.dart | 2 +- .../language/sync_star_semantics_bench.dart | 2 +- .../lib/layout/text_intrinsic_bench.dart | 2 +- .../lib/stocks/animation_bench.dart | 33 ++++-- .../lib/stocks/build_bench.dart | 2 +- .../lib/stocks/build_bench_profiled.dart | 2 +- .../lib/stocks/layout_bench.dart | 2 +- .../microbenchmarks/lib/ui/image_bench.dart | 2 +- dev/devicelab/lib/microbenchmarks.dart | 43 +++---- dev/devicelab/lib/tasks/microbenchmarks.dart | 42 +++---- 26 files changed, 105 insertions(+), 201 deletions(-) delete mode 100644 dev/benchmarks/microbenchmarks/lib/benchmark_binding.dart delete mode 100644 dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart diff --git a/dev/benchmarks/microbenchmarks/README.md b/dev/benchmarks/microbenchmarks/README.md index a9fcf49d7d..157869ad10 100644 --- a/dev/benchmarks/microbenchmarks/README.md +++ b/dev/benchmarks/microbenchmarks/README.md @@ -1,17 +1,22 @@ # microbenchmarks -To run these benchmarks on a device, first run `flutter logs` in one -window to see the device logs, then, in a different window, run: +To run these benchmarks on a device, first run `flutter logs' in one +window to see the device logs, then, in a different window, run any of +these: ```sh -flutter run -d $DEVICE_ID --profile lib/benchmark_collection.dart +flutter run --release lib/gestures/velocity_tracker_bench.dart +flutter run --release lib/gestures/gesture_detector_bench.dart +flutter run --release lib/stocks/animation_bench.dart +flutter run --release lib/stocks/build_bench.dart +flutter run --release lib/stocks/layout_bench.dart ``` The results should be in the device logs. -## Avoid changing names of the benchmarks +### Avoid changing names of the benchmarks Each microbenchmark is identified by a name, for example, -"catmullrom_transform_iteration". Changing the name passed to `BenchmarkResultPrinter.addResult` -will effectively remove the old benchmark and create a new one, +"catmullrom_transform_iteration". Changing the name of an existing +microbenchmarks will effectively remove the old benchmark and create a new one, losing the historical data associated with the old benchmark in the process. diff --git a/dev/benchmarks/microbenchmarks/lib/benchmark_binding.dart b/dev/benchmarks/microbenchmarks/lib/benchmark_binding.dart deleted file mode 100644 index 8ca2ba9d9e..0000000000 --- a/dev/benchmarks/microbenchmarks/lib/benchmark_binding.dart +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:flutter_test/flutter_test.dart'; - -class BenchmarkingBinding extends LiveTestWidgetsFlutterBinding { - BenchmarkingBinding(); - - final Stopwatch drawFrameWatch = Stopwatch(); - - @override - void handleBeginFrame(Duration? rawTimeStamp) { - drawFrameWatch.start(); - super.handleBeginFrame(rawTimeStamp); - } - - @override - void handleDrawFrame() { - super.handleDrawFrame(); - drawFrameWatch.stop(); - } -} diff --git a/dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart b/dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart deleted file mode 100644 index acb280c4f0..0000000000 --- a/dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:io'; - -import 'package:flutter/material.dart'; -import 'package:flutter/scheduler.dart'; -import 'package:flutter_test/flutter_test.dart'; - -import 'benchmark_binding.dart'; -import 'foundation/all_elements_bench.dart' as all_elements_bench; -import 'foundation/change_notifier_bench.dart' as change_notifier_bench; -import 'foundation/clamp.dart' as clamp; -import 'foundation/decode_and_parse_asset_manifest.dart' - as decode_and_parse_asset_manifest; -import 'foundation/platform_asset_bundle.dart' as platform_asset_bundle; -import 'foundation/standard_message_codec_bench.dart' - as standard_message_codec_bench; -import 'foundation/standard_method_codec_bench.dart' - as standard_method_codec_bench; -import 'foundation/timeline_bench.dart' as timeline_bench; -import 'geometry/matrix_utils_transform_bench.dart' - as matrix_utils_transform_bench; -import 'geometry/rrect_contains_bench.dart' as rrect_contains_bench; -import 'gestures/gesture_detector_bench.dart' as gesture_detector_bench; -import 'gestures/velocity_tracker_bench.dart' as velocity_tracker_bench; -import 'language/compute_bench.dart' as compute_bench; -import 'language/sync_star_bench.dart' as sync_star_bench; -import 'language/sync_star_semantics_bench.dart' as sync_star_semantics_bench; -import 'layout/text_intrinsic_bench.dart' as text_intrinsic_bench; -import 'stocks/animation_bench.dart' as animation_bench; -import 'stocks/build_bench.dart' as build_bench; -import 'stocks/build_bench_profiled.dart' as build_bench_profiled; -import 'stocks/layout_bench.dart' as layout_bench; -import 'ui/image_bench.dart' as image_bench; - -typedef Benchmark = (String name, Future Function() value); - -Future main() async { - assert(false, - "Don't run benchmarks in debug mode! Use 'flutter run --release'."); - - // BenchmarkingBinding is used by animation_bench, providing a simple - // stopwatch interface over rendering. Lifting it here makes all - // benchmarks run together. - final BenchmarkingBinding binding = BenchmarkingBinding(); - final List benchmarks = [ - ('foundation/change_notifier_bench.dart', change_notifier_bench.execute), - ('foundation/clamp.dart', clamp.execute), - ('foundation/platform_asset_bundle.dart', platform_asset_bundle.execute), - ( - 'foundation/standard_message_codec_bench.dart', - standard_message_codec_bench.execute - ), - ( - 'foundation/standard_method_codec_bench.dart', - standard_method_codec_bench.execute - ), - ('foundation/timeline_bench.dart', timeline_bench.execute), - ( - 'foundation/decode_and_parse_asset_manifest.dart', - decode_and_parse_asset_manifest.execute - ), - ( - 'geometry/matrix_utils_transform_bench.dart', - matrix_utils_transform_bench.execute - ), - ('geometry/rrect_contains_bench.dart', rrect_contains_bench.execute), - ('gestures/gesture_detector_bench.dart', gesture_detector_bench.execute), - ('gestures/velocity_tracker_bench.dart', velocity_tracker_bench.execute), - ('language/compute_bench.dart', compute_bench.execute), - ('language/sync_star_bench.dart', sync_star_bench.execute), - ( - 'language/sync_star_semantics_bench.dart', - sync_star_semantics_bench.execute - ), - ('stocks/animation_bench.dart', () => animation_bench.execute(binding)), - ('stocks/build_bench.dart', build_bench.execute), - ('stocks/build_bench_profiled.dart', build_bench_profiled.execute), - ('stocks/layout_bench.dart', layout_bench.execute), - ('ui/image_bench.dart', image_bench.execute), - ('layout/text_intrinsic_bench.dart', text_intrinsic_bench.execute), - ( - 'foundation/all_elements_bench.dart', - () async { - binding.framePolicy = - LiveTestWidgetsFlutterBindingFramePolicy.fullyLive; - runApp(const SizedBox.shrink()); // ensure dispose - await SchedulerBinding.instance.endOfFrame; - all_elements_bench.execute(); - } - ), - ]; - - print('╡ ••• Running microbenchmarks ••• ╞'); - - for (final Benchmark mark in benchmarks) { - // Reset the frame policy to default - each test can set it on their own. - binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fadePointers; - print('╡ ••• Running ${mark.$1} ••• ╞'); - await mark.$2(); - } - - print('\n\n╡ ••• Done ••• ╞\n\n'); - exit(0); -} diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/all_elements_bench.dart b/dev/benchmarks/microbenchmarks/lib/foundation/all_elements_bench.dart index fcfa798d69..23dd6d5169 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/all_elements_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/all_elements_bench.dart @@ -11,7 +11,8 @@ import '../common.dart'; const int _kNumIters = 10000; -Future execute() async { +Future main() async { + assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); runApp(MaterialApp( home: Scaffold( body: GridView.count( diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/change_notifier_bench.dart b/dev/benchmarks/microbenchmarks/lib/foundation/change_notifier_bench.dart index 3dcc8649bc..0eedaef66b 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/change_notifier_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/change_notifier_bench.dart @@ -10,7 +10,7 @@ const int _kNumIterations = 65536; const int _kNumWarmUp = 100; const int _kScale = 1000; -Future execute() async { +void main() { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); // In the following benchmarks, we won't remove the listeners when we don't diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/clamp.dart b/dev/benchmarks/microbenchmarks/lib/foundation/clamp.dart index 3b1184da00..1a933193ab 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/clamp.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/clamp.dart @@ -9,7 +9,7 @@ import '../common.dart'; const int _kBatchSize = 100000; const int _kNumIterations = 1000; -Future execute() async { +void main() { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/decode_and_parse_asset_manifest.dart b/dev/benchmarks/microbenchmarks/lib/foundation/decode_and_parse_asset_manifest.dart index ccc3901a75..5082abcf2f 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/decode_and_parse_asset_manifest.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/decode_and_parse_asset_manifest.dart @@ -9,7 +9,7 @@ import '../common.dart'; const int _kNumIterations = 1000; -Future execute() async { +void main() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/platform_asset_bundle.dart b/dev/benchmarks/microbenchmarks/lib/foundation/platform_asset_bundle.dart index e55d6183b4..4f2b172a10 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/platform_asset_bundle.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/platform_asset_bundle.dart @@ -10,7 +10,7 @@ import '../common.dart'; const int _kBatchSize = 100; const int _kNumIterations = 100; -Future execute() async { +void main() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/standard_message_codec_bench.dart b/dev/benchmarks/microbenchmarks/lib/foundation/standard_message_codec_bench.dart index daae70467e..58aaa0f60f 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/standard_message_codec_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/standard_message_codec_bench.dart @@ -8,7 +8,7 @@ import '../common.dart'; const int _kNumIterations = 100000; -Future execute() async { +void main() { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/standard_method_codec_bench.dart b/dev/benchmarks/microbenchmarks/lib/foundation/standard_method_codec_bench.dart index a274befcf7..6ee3b64be4 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/standard_method_codec_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/standard_method_codec_bench.dart @@ -8,7 +8,7 @@ import '../common.dart'; const int _kNumIterations = 100000; -Future execute() async { +void main() { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/foundation/timeline_bench.dart b/dev/benchmarks/microbenchmarks/lib/foundation/timeline_bench.dart index 84b7fb8cbd..7c9eb4a396 100644 --- a/dev/benchmarks/microbenchmarks/lib/foundation/timeline_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/foundation/timeline_bench.dart @@ -8,7 +8,7 @@ import '../common.dart'; const int _kNumIterations = 10000; -Future execute() async { +void main() { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); diff --git a/dev/benchmarks/microbenchmarks/lib/geometry/matrix_utils_transform_bench.dart b/dev/benchmarks/microbenchmarks/lib/geometry/matrix_utils_transform_bench.dart index b0096369d7..35b9fecae9 100644 --- a/dev/benchmarks/microbenchmarks/lib/geometry/matrix_utils_transform_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/geometry/matrix_utils_transform_bench.dart @@ -11,7 +11,7 @@ import '../common.dart'; const int _kNumIterations = 10000000; const int _kNumWarmUp = 100000; -Future execute() async { +void main() { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); print('MatrixUtils.transformRect and .transformPoint benchmark...'); diff --git a/dev/benchmarks/microbenchmarks/lib/geometry/rrect_contains_bench.dart b/dev/benchmarks/microbenchmarks/lib/geometry/rrect_contains_bench.dart index de7429de64..4afd3c976e 100644 --- a/dev/benchmarks/microbenchmarks/lib/geometry/rrect_contains_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/geometry/rrect_contains_bench.dart @@ -8,7 +8,7 @@ import '../common.dart'; const int _kNumIters = 10000; -Future execute() async { +void main() { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final Stopwatch watch = Stopwatch(); print('RRect contains benchmark...'); diff --git a/dev/benchmarks/microbenchmarks/lib/gestures/gesture_detector_bench.dart b/dev/benchmarks/microbenchmarks/lib/gestures/gesture_detector_bench.dart index 6ee61e3afe..a5ca499555 100644 --- a/dev/benchmarks/microbenchmarks/lib/gestures/gesture_detector_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/gestures/gesture_detector_bench.dart @@ -10,7 +10,7 @@ import 'apps/button_matrix_app.dart' as button_matrix; const int _kNumWarmUpIters = 20; const int _kNumIters = 300; -Future execute() async { +Future main() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final Stopwatch watch = Stopwatch(); print('GestureDetector semantics benchmark...'); diff --git a/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart b/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart index 996c306c25..dcf4064525 100644 --- a/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart @@ -17,7 +17,7 @@ class TrackerBenchmark { final String name; } -Future execute() async { +Future main() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); final List benchmarks = [ diff --git a/dev/benchmarks/microbenchmarks/lib/language/compute_bench.dart b/dev/benchmarks/microbenchmarks/lib/language/compute_bench.dart index 59e08487b7..36e8ef7527 100644 --- a/dev/benchmarks/microbenchmarks/lib/language/compute_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/language/compute_bench.dart @@ -23,7 +23,7 @@ List test(int length) { (int index) => Data(index * index)); } -Future execute() async { +Future main() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); // Warm up lap diff --git a/dev/benchmarks/microbenchmarks/lib/language/sync_star_bench.dart b/dev/benchmarks/microbenchmarks/lib/language/sync_star_bench.dart index 41123e8e90..2b34dcd6b4 100644 --- a/dev/benchmarks/microbenchmarks/lib/language/sync_star_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/language/sync_star_bench.dart @@ -7,7 +7,7 @@ import '../common.dart'; const int _kNumIterations = 1000; const int _kNumWarmUp = 100; -Future execute() async { +void main() { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); // Warm up lap diff --git a/dev/benchmarks/microbenchmarks/lib/language/sync_star_semantics_bench.dart b/dev/benchmarks/microbenchmarks/lib/language/sync_star_semantics_bench.dart index 993d98b9b2..ab8d098337 100644 --- a/dev/benchmarks/microbenchmarks/lib/language/sync_star_semantics_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/language/sync_star_semantics_bench.dart @@ -9,7 +9,7 @@ import '../common.dart'; const int _kNumIterations = 1000; const int _kNumWarmUp = 100; -Future execute() async { +void main() { final List words = 'Lorem Ipsum is simply dummy text of the printing and' " typesetting industry. Lorem Ipsum has been the industry's" ' standard dummy text ever since the 1500s, when an unknown' diff --git a/dev/benchmarks/microbenchmarks/lib/layout/text_intrinsic_bench.dart b/dev/benchmarks/microbenchmarks/lib/layout/text_intrinsic_bench.dart index b43bea0b1a..95da4a18fd 100644 --- a/dev/benchmarks/microbenchmarks/lib/layout/text_intrinsic_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/layout/text_intrinsic_bench.dart @@ -20,7 +20,7 @@ final Widget intrinsicTextHeight = Directionality( ), ); -Future execute() async { +Future main() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); // We control the framePolicy below to prevent us from scheduling frames in diff --git a/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart b/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart index ee5498019a..d9fcd4376b 100644 --- a/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart @@ -7,16 +7,35 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:stocks/main.dart' as stocks; import 'package:stocks/stock_data.dart' as stock_data; -import '../benchmark_binding.dart'; import '../common.dart'; const Duration kBenchmarkTime = Duration(seconds: 15); -Future execute(BenchmarkingBinding binding) async { +class BenchmarkingBinding extends LiveTestWidgetsFlutterBinding { + BenchmarkingBinding(this.stopwatch); + + final Stopwatch stopwatch; + + @override + void handleBeginFrame(Duration? rawTimeStamp) { + stopwatch.start(); + super.handleBeginFrame(rawTimeStamp); + } + + @override + void handleDrawFrame() { + super.handleDrawFrame(); + stopwatch.stop(); + } +} + +Future main() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); stock_data.StockData.actuallyFetchData = false; final Stopwatch wallClockWatch = Stopwatch(); + final Stopwatch cpuWatch = Stopwatch(); + BenchmarkingBinding(cpuWatch); int totalOpenFrameElapsedMicroseconds = 0; int totalOpenIterationCount = 0; @@ -33,27 +52,27 @@ Future execute(BenchmarkingBinding binding) async { bool drawerIsOpen = false; wallClockWatch.start(); while (wallClockWatch.elapsed < kBenchmarkTime) { - binding.drawFrameWatch.reset(); + cpuWatch.reset(); if (drawerIsOpen) { await tester.tapAt(const Offset(780.0, 250.0)); // Close drawer await tester.pump(); totalCloseIterationCount += 1; - totalCloseFrameElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds; + totalCloseFrameElapsedMicroseconds += cpuWatch.elapsedMicroseconds; } else { await tester.tapAt(const Offset(20.0, 50.0)); // Open drawer await tester.pump(); totalOpenIterationCount += 1; - totalOpenFrameElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds; + totalOpenFrameElapsedMicroseconds += cpuWatch.elapsedMicroseconds; } drawerIsOpen = !drawerIsOpen; // Time how long each frame takes - binding.drawFrameWatch.reset(); + cpuWatch.reset(); while (SchedulerBinding.instance.hasScheduledFrame) { await tester.pump(); totalSubsequentFramesIterationCount += 1; } - totalSubsequentFramesElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds; + totalSubsequentFramesElapsedMicroseconds += cpuWatch.elapsedMicroseconds; } }); diff --git a/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart b/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart index 33eb5bd1d1..6b868ba169 100644 --- a/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart @@ -55,7 +55,7 @@ Future> runBuildBenchmark() async { return values; } -Future execute() async { +Future main() async { final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); printer.addResultStatistics( description: 'Stock build', diff --git a/dev/benchmarks/microbenchmarks/lib/stocks/build_bench_profiled.dart b/dev/benchmarks/microbenchmarks/lib/stocks/build_bench_profiled.dart index 2774d461fe..8f48ca7ee1 100644 --- a/dev/benchmarks/microbenchmarks/lib/stocks/build_bench_profiled.dart +++ b/dev/benchmarks/microbenchmarks/lib/stocks/build_bench_profiled.dart @@ -8,7 +8,7 @@ import 'package:flutter_test/flutter_test.dart'; import '../common.dart'; import 'build_bench.dart'; -Future execute() async { +Future main() async { debugProfileBuildsEnabledUserWidgets = true; final BenchmarkResultPrinter printer = BenchmarkResultPrinter(); printer.addResultStatistics( diff --git a/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart b/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart index e638ff7382..9d72b87aa3 100644 --- a/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart @@ -12,7 +12,7 @@ import '../common.dart'; const Duration kBenchmarkTime = Duration(seconds: 15); -Future execute() async { +Future main() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); stock_data.StockData.actuallyFetchData = false; diff --git a/dev/benchmarks/microbenchmarks/lib/ui/image_bench.dart b/dev/benchmarks/microbenchmarks/lib/ui/image_bench.dart index 96a7e4ea5b..2b807acf3c 100644 --- a/dev/benchmarks/microbenchmarks/lib/ui/image_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/ui/image_bench.dart @@ -78,7 +78,7 @@ const List assets = [ // Measures the time it takes to load a fixed number of assets into an // immutable buffer to later be decoded by skia. -Future execute() async { +Future main() async { assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'."); final Stopwatch watch = Stopwatch(); diff --git a/dev/devicelab/lib/microbenchmarks.dart b/dev/devicelab/lib/microbenchmarks.dart index a218a35bc0..451be27b1a 100644 --- a/dev/devicelab/lib/microbenchmarks.dart +++ b/dev/devicelab/lib/microbenchmarks.dart @@ -14,8 +14,6 @@ Future> readJsonResults(Process process) { const String jsonStart = '================ RESULTS ================'; const String jsonEnd = '================ FORMATTED =============='; const String jsonPrefix = ':::JSON:::'; - const String testComplete = '╡ ••• Done ••• ╞'; - bool jsonStarted = false; final StringBuffer jsonBuf = StringBuffer(); final Completer> completer = Completer>(); @@ -27,9 +25,8 @@ Future> readJsonResults(Process process) { stderr.writeln('[STDERR] $line'); }); - final List collectedJson = []; - bool processWasKilledIntentionally = false; + bool resultsHaveBeenParsed = false; final StreamSubscription stdoutSub = process.stdout .transform(const Utf8Decoder()) .transform(const LineSplitter()) @@ -41,40 +38,48 @@ Future> readJsonResults(Process process) { return; } - if (line.contains(testComplete)) { + if (jsonStarted && line.contains(jsonEnd)) { + final String jsonOutput = jsonBuf.toString(); + + // If we end up here and have already parsed the results, it suggests that + // we have received output from another test because our `flutter run` + // process did not terminate correctly. + // https://github.com/flutter/flutter/issues/19096#issuecomment-402756549 + if (resultsHaveBeenParsed) { + throw 'Additional JSON was received after results has already been ' + 'processed. This suggests the `flutter run` process may have lived ' + 'past the end of our test and collected additional output from the ' + 'next test.\n\n' + 'The JSON below contains all collected output, including both from ' + 'the original test and what followed.\n\n' + '$jsonOutput'; + } + + jsonStarted = false; processWasKilledIntentionally = true; + resultsHaveBeenParsed = true; // Sending a SIGINT/SIGTERM to the process here isn't reliable because [process] is // the shell (flutter is a shell script) and doesn't pass the signal on. // Sending a `q` is an instruction to quit using the console runner. // See https://github.com/flutter/flutter/issues/19208 process.stdin.write('q'); await process.stdin.flush(); + // Give the process a couple of seconds to exit and run shutdown hooks // before sending kill signal. // TODO(fujino): https://github.com/flutter/flutter/issues/134566 await Future.delayed(const Duration(seconds: 2)); + // Also send a kill signal in case the `q` above didn't work. process.kill(ProcessSignal.sigint); try { - final Map results = - Map.from({ - for (final String data in collectedJson) - ...json.decode(data) as Map - }); - completer.complete(results); + completer.complete(Map.from(json.decode(jsonOutput) as Map)); } catch (ex) { - completer.completeError( - 'Decoding JSON failed ($ex). JSON strings where: $collectedJson'); + completer.completeError('Decoding JSON failed ($ex). JSON string was: $jsonOutput'); } return; } - if (jsonStarted && line.contains(jsonEnd)) { - collectedJson.add(jsonBuf.toString().trim()); - jsonBuf.clear(); - jsonStarted = false; - } - if (jsonStarted && line.contains(jsonPrefix)) { jsonBuf.writeln(line.substring(line.indexOf(jsonPrefix) + jsonPrefix.length)); } diff --git a/dev/devicelab/lib/tasks/microbenchmarks.dart b/dev/devicelab/lib/tasks/microbenchmarks.dart index b5e17f3c22..aa38ab9a57 100644 --- a/dev/devicelab/lib/tasks/microbenchmarks.dart +++ b/dev/devicelab/lib/tasks/microbenchmarks.dart @@ -24,27 +24,11 @@ TaskFunction createMicrobenchmarkTask({ await device.unlock(); await device.clearLogs(); - final Directory appDir = - dir(path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks')); - - // Hard-uninstall any prior apps. - await inDirectory(appDir, () async { - section('Uninstall previous microbenchmarks app'); - await flutter( - 'install', - options: [ - '-v', - '--uninstall-only', - '-d', - device.deviceId, - ], - ); - }); - Future> runMicrobench(String benchmarkPath) async { Future> run() async { print('Running $benchmarkPath'); - + final Directory appDir = dir( + path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks')); final Process flutterProcess = await inDirectory(appDir, () async { final List options = [ '-v', @@ -70,7 +54,27 @@ TaskFunction createMicrobenchmarkTask({ } final Map allResults = { - ...await runMicrobench('lib/benchmark_collection.dart'), + ...await runMicrobench('lib/foundation/all_elements_bench.dart'), + ...await runMicrobench('lib/foundation/change_notifier_bench.dart'), + ...await runMicrobench('lib/foundation/clamp.dart'), + ...await runMicrobench('lib/foundation/platform_asset_bundle.dart'), + ...await runMicrobench('lib/foundation/standard_message_codec_bench.dart'), + ...await runMicrobench('lib/foundation/standard_method_codec_bench.dart'), + ...await runMicrobench('lib/foundation/timeline_bench.dart'), + ...await runMicrobench('lib/foundation/decode_and_parse_asset_manifest.dart'), + ...await runMicrobench('lib/geometry/matrix_utils_transform_bench.dart'), + ...await runMicrobench('lib/geometry/rrect_contains_bench.dart'), + ...await runMicrobench('lib/gestures/gesture_detector_bench.dart'), + ...await runMicrobench('lib/gestures/velocity_tracker_bench.dart'), + ...await runMicrobench('lib/language/compute_bench.dart'), + ...await runMicrobench('lib/language/sync_star_bench.dart'), + ...await runMicrobench('lib/language/sync_star_semantics_bench.dart'), + ...await runMicrobench('lib/stocks/animation_bench.dart'), + ...await runMicrobench('lib/stocks/build_bench_profiled.dart'), + ...await runMicrobench('lib/stocks/build_bench.dart'), + ...await runMicrobench('lib/stocks/layout_bench.dart'), + ...await runMicrobench('lib/ui/image_bench.dart'), + ...await runMicrobench('lib/layout/text_intrinsic_bench.dart'), }; return TaskResult.success(allResults,