diff --git a/dev/benchmarks/microbenchmarks/README.md b/dev/benchmarks/microbenchmarks/README.md index a9fcf49d7d..a30fc0b15a 100644 --- a/dev/benchmarks/microbenchmarks/README.md +++ b/dev/benchmarks/microbenchmarks/README.md @@ -7,6 +7,19 @@ window to see the device logs, then, in a different window, run: flutter run -d $DEVICE_ID --profile lib/benchmark_collection.dart ``` +To run a subset of tests: + +```shell +flutter run -d $DEVICE_ID --profile lib/benchmark_collection.dart --dart-define=tests=foundation/change_notifier_bench.dart,language/sync_star_bench.dart +``` + +To specify a seed value for shuffling tests: + +```shell +flutter run -d $DEVICE_ID --profile lib/benchmark_collection.dart --dart-define=seed=12345 +``` + + The results should be in the device logs. ## Avoid changing names of the benchmarks diff --git a/dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart b/dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart index acb280c4f0..28dff09ded 100644 --- a/dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart +++ b/dev/benchmarks/microbenchmarks/lib/benchmark_collection.dart @@ -3,7 +3,9 @@ // found in the LICENSE file. import 'dart:io'; +import 'dart:math'; +import 'package:args/args.dart'; import 'package:flutter/material.dart'; import 'package:flutter/scheduler.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -93,9 +95,44 @@ Future main() async { ), ]; - print('╡ ••• Running microbenchmarks ••• ╞'); + // Parses the optional compile-time dart variables; we can't have + // arguments passed in to main. + final ArgParser parser = ArgParser(); + final List allowed = benchmarks.map((Benchmark e) => e.$1).toList(); + parser.addMultiOption( + 'tests', + abbr: 't', + defaultsTo: allowed, + allowed: allowed, + help: 'selected tests to run', + ); + parser.addOption( + 'seed', + defaultsTo: '12345', + help: 'selects seed to sort tests by', + ); + final List mainArgs = []; + const String testArgs = String.fromEnvironment('tests'); + if (testArgs.isNotEmpty) { + mainArgs.addAll(['--tests', testArgs]); + print('╡ ••• environment test override: $testArgs ••• ╞'); + } + const String seedArgs = String.fromEnvironment('seed'); + if (seedArgs.isNotEmpty) { + mainArgs.addAll(['--seed', seedArgs]); + print('╡ ••• environment seed override: $seedArgs ••• ╞'); + } + final ArgResults results = parser.parse(mainArgs); + final List selectedTests = results.multiOption('tests'); - for (final Benchmark mark in benchmarks) { + // Shuffle the tests becauase we don't want order dependent tests. + // It is the responsibily of the infra to tell us what the seed value is, + // in case we want to have the seed stable for some time period. + final List tests = benchmarks.where((Benchmark e) => selectedTests.contains(e.$1)).toList(); + tests.shuffle(Random(int.parse(results.option('seed')!))); + + print('╡ ••• Running microbenchmarks ••• ╞'); + for (final Benchmark mark in tests) { // Reset the frame policy to default - each test can set it on their own. binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fadePointers; print('╡ ••• Running ${mark.$1} ••• ╞'); diff --git a/dev/devicelab/lib/tasks/microbenchmarks.dart b/dev/devicelab/lib/tasks/microbenchmarks.dart index b5e17f3c22..4d2cf67b68 100644 --- a/dev/devicelab/lib/tasks/microbenchmarks.dart +++ b/dev/devicelab/lib/tasks/microbenchmarks.dart @@ -19,6 +19,10 @@ TaskFunction createMicrobenchmarkTask({ bool? enableImpeller, Map environment = const {}, }) { + + // Generate a seed for this test stable around the date. + final int seed = DateTime.now().toUtc().subtract(const Duration(hours: 7)).hashCode; + return () async { final Device device = await devices.workingDevice; await device.unlock(); @@ -43,7 +47,7 @@ TaskFunction createMicrobenchmarkTask({ Future> runMicrobench(String benchmarkPath) async { Future> run() async { - print('Running $benchmarkPath'); + print('Running $benchmarkPath with seed $seed'); final Process flutterProcess = await inDirectory(appDir, () async { final List options = [ @@ -55,6 +59,7 @@ TaskFunction createMicrobenchmarkTask({ if (enableImpeller != null && !enableImpeller) '--no-enable-impeller', '-d', device.deviceId, + '--dart-define=seed=$seed', benchmarkPath, ]; return startFlutter(