From 1df639b4320baa544f3cecac08944fb00fe61a0d Mon Sep 17 00:00:00 2001 From: Yegor Date: Mon, 9 Jan 2017 14:57:14 -0800 Subject: [PATCH] add microbenchmarks to devicelab (fixes #7081) (#7396) --- .../microbenchmarks/lib/common.dart | 79 ++++++++++++++++ .../lib/gestures/velocity_tracker_bench.dart | 12 ++- .../lib/stocks/animation_bench.dart | 32 ++++++- .../lib/stocks/build_bench.dart | 11 ++- .../lib/stocks/layout_bench.dart | 13 ++- dev/benchmarks/microbenchmarks/pubspec.yaml | 1 + dev/devicelab/bin/tasks/microbenchmarks.dart | 90 +++++++++++++++++++ dev/devicelab/manifest.yaml | 6 ++ 8 files changed, 235 insertions(+), 9 deletions(-) create mode 100644 dev/benchmarks/microbenchmarks/lib/common.dart create mode 100644 dev/devicelab/bin/tasks/microbenchmarks.dart diff --git a/dev/benchmarks/microbenchmarks/lib/common.dart b/dev/benchmarks/microbenchmarks/lib/common.dart new file mode 100644 index 0000000000..7a9796e47f --- /dev/null +++ b/dev/benchmarks/microbenchmarks/lib/common.dart @@ -0,0 +1,79 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert' show JsonEncoder; + +import 'package:meta/meta.dart'; + +/// This class knows how to format benchmark results for machine and human +/// consumption. +/// + +/// Example: +/// +/// BenchmarkResultPrinter printer = new BenchmarkResultPrinter(); +/// printer.add( +/// description: 'Average frame time', +/// value: averageFrameTime, +/// unit: 'ms', +/// name: 'average_frame_time', +/// ); +/// printer.printToStdout(); +/// +class BenchmarkResultPrinter { + + final List<_BenchmarkResult> _results = <_BenchmarkResult>[]; + + /// Adds a benchmark result to the list of results. + /// + /// [description] is a human-readable description of the result. [value] is a + /// result value. [unit] is the unit of measurement, such as "ms", "km", "h". + /// [name] is a computer-readable name of the result used as a key in the JSON + /// serialization of the results. + void addResult({ @required String description, @required double value, @required String unit, @required String name }) { + _results.add(new _BenchmarkResult(description, value, unit, name)); + } + + /// Prints the results added via [addResult] to standard output, once as JSON + /// for computer consumption and once formatted as plain text for humans. + void printToStdout() { + // IMPORTANT: keep these values in sync with dev/devicelab/bin/tasks/microbenchmarks.dart + print('================ RESULTS ================'); + print(_printJson()); + print('================ FORMATTED =============='); + print(_printPlainText()); + } + + String _printJson() { + const JsonEncoder encoder = const JsonEncoder.withIndent(' '); + return encoder.convert(new Map.fromIterable(_results, + key: (_BenchmarkResult result) => result.name, + value: (_BenchmarkResult result) => result.value, + )); + } + + String _printPlainText() { + StringBuffer buf = new StringBuffer(); + for (_BenchmarkResult result in _results) { + buf.writeln('${result.description}: ${result.value.toStringAsFixed(1)} ${result.unit}'); + } + return buf.toString(); + } +} + +class _BenchmarkResult { + _BenchmarkResult(this.description, this.value, this.unit, this.name); + + /// Human-readable description of the result, e.g. "Average frame time". + final String description; + + /// Result value that in agreement with [unit]. + final double value; + + /// Unit of measurement that is in agreement with [value]. + final String unit; + + /// Computer-readable name of the result. + final String name; +} diff --git a/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart b/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart index f9344a0147..4c681cbca1 100644 --- a/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart @@ -6,6 +6,8 @@ import 'dart:io'; import 'package:flutter/gestures.dart'; import 'data/velocity_tracker_data.dart'; +import '../common.dart'; + const int _kNumIters = 10000; void main() { @@ -22,6 +24,14 @@ void main() { } } watch.stop(); - print('Velocity tracker: ${(watch.elapsedMicroseconds / _kNumIters).toStringAsFixed(1)}µs per iteration'); + + BenchmarkResultPrinter printer = new BenchmarkResultPrinter(); + printer.addResult( + description: 'Velocity tracker', + value: watch.elapsedMicroseconds / _kNumIters, + unit: 'µs per iteration', + name: 'velocity_tracker_iteration', + ); + printer.printToStdout(); exit(0); } diff --git a/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart b/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart index 717b123eb3..f6f74f4638 100644 --- a/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart @@ -11,6 +11,8 @@ import 'package:flutter/scheduler.dart'; import 'package:stocks/main.dart' as stocks; import 'package:stocks/stock_data.dart' as stock_data; +import '../common.dart'; + const Duration kBenchmarkTime = const Duration(seconds: 15); class BenchmarkingBinding extends LiveTestWidgetsFlutterBinding { @@ -73,10 +75,32 @@ Future main() async { } }); - print('Stock animation (ran for ${(wallClockWatch.elapsedMicroseconds / (1000 * 1000)).toStringAsFixed(1)}s):'); - print(' Opening first frame average time: ${(totalOpenFrameElapsedMicroseconds / (totalOpenIterationCount)).toStringAsFixed(1)}µs per frame ($totalOpenIterationCount frames)'); - print(' Closing first frame average time: ${(totalCloseFrameElapsedMicroseconds / (totalCloseIterationCount)).toStringAsFixed(1)}µs per frame ($totalCloseIterationCount frames)'); - print(' Subsequent frames average time: ${(totalSubsequentFramesElapsedMicroseconds / (totalSubsequentFramesIterationCount)).toStringAsFixed(1)}µs per frame ($totalSubsequentFramesIterationCount frames)'); + BenchmarkResultPrinter printer = new BenchmarkResultPrinter(); + printer.addResult( + description: 'Stock animation', + value: wallClockWatch.elapsedMicroseconds / (1000 * 1000), + unit: 's', + name: 'stock_animation_total_run_time', + ); + printer.addResult( + description: ' Opening first frame average time', + value: totalOpenFrameElapsedMicroseconds / totalOpenIterationCount, + unit: 'µs per frame ($totalOpenIterationCount frames)', + name: 'stock_animation_open_first_frame_average', + ); + printer.addResult( + description: ' Closing first frame average time', + value: totalCloseFrameElapsedMicroseconds / totalCloseIterationCount, + unit: 'µs per frame ($totalCloseIterationCount frames)', + name: 'stock_animation_close_first_frame_average', + ); + printer.addResult( + description: ' Subsequent frames average time', + value: totalSubsequentFramesElapsedMicroseconds / totalSubsequentFramesIterationCount, + unit: 'µs per frame ($totalSubsequentFramesIterationCount frames)', + name: 'stock_animation_subsequent_frame_average', + ); + printer.printToStdout(); exit(0); } diff --git a/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart b/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart index 709487d89a..791760a55a 100644 --- a/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart @@ -6,6 +6,8 @@ import 'package:flutter/rendering.dart'; import 'package:stocks/main.dart' as stocks; import 'package:stocks/stock_data.dart' as stock_data; +import '../common.dart'; + const Duration kBenchmarkTime = const Duration(seconds: 15); Future main() async { @@ -36,6 +38,13 @@ Future main() async { watch.stop(); }); - print('Stock build: ${(watch.elapsedMicroseconds / iterations).toStringAsFixed(1)}µs per iteration'); + BenchmarkResultPrinter printer = new BenchmarkResultPrinter(); + printer.addResult( + description: 'Stock build', + value: watch.elapsedMicroseconds / iterations, + unit: 'µs per iteration', + name: 'stock_build_iteration', + ); + printer.printToStdout(); exit(0); } diff --git a/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart b/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart index 20902fe567..b19036d308 100644 --- a/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart +++ b/dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart @@ -6,10 +6,10 @@ import 'package:flutter/rendering.dart'; import 'package:stocks/main.dart' as stocks; import 'package:stocks/stock_data.dart' as stock_data; +import '../common.dart'; + const Duration kBenchmarkTime = const Duration(seconds: 15); - - Future main() async { stock_data.StockDataFetcher.actuallyFetchData = false; @@ -37,6 +37,13 @@ Future main() async { watch.stop(); }); - print('Stock layout: ${(watch.elapsedMicroseconds / iterations).toStringAsFixed(1)}µs per iteration'); + BenchmarkResultPrinter printer = new BenchmarkResultPrinter(); + printer.addResult( + description: 'Stock layout', + value: watch.elapsedMicroseconds / iterations, + unit: 'µs per iteration', + name: 'stock_layout_iteration', + ); + printer.printToStdout(); exit(0); } diff --git a/dev/benchmarks/microbenchmarks/pubspec.yaml b/dev/benchmarks/microbenchmarks/pubspec.yaml index 007fc940bf..da5e4561fe 100644 --- a/dev/benchmarks/microbenchmarks/pubspec.yaml +++ b/dev/benchmarks/microbenchmarks/pubspec.yaml @@ -1,6 +1,7 @@ name: microbenchmarks description: A new flutter project. dependencies: + meta: ^1.0.3 flutter: sdk: flutter flutter_test: diff --git a/dev/devicelab/bin/tasks/microbenchmarks.dart b/dev/devicelab/bin/tasks/microbenchmarks.dart new file mode 100644 index 0000000000..020a1cc1c0 --- /dev/null +++ b/dev/devicelab/bin/tasks/microbenchmarks.dart @@ -0,0 +1,90 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as path; + +import 'package:flutter_devicelab/framework/adb.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/framework/utils.dart'; + +/// Runs benchmarks in `dev/benchmarks/microbenchmarks` in the device lab and +/// reports results to the dashboard. +Future main() async { + await task(() async { + Device device = await devices.workingDevice; + await device.unlock(); + + Future> _runMicrobench(String benchmarkPath) async { + print('Running $benchmarkPath'); + Directory appDir = dir(path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks')); + Process flutterProcess = await inDirectory(appDir, () async { + return await _startFlutter( + options: [ + '--release', + '-d', + device.deviceId, + benchmarkPath, + ], + canFail: false, + ); + }); + + return await _readJsonResults(flutterProcess); + } + + Map allResults = {}; + allResults.addAll(await _runMicrobench('lib/stocks/layout_bench.dart')); + allResults.addAll(await _runMicrobench('lib/stocks/build_bench.dart')); + allResults.addAll(await _runMicrobench('lib/stocks/animation_bench.dart')); + allResults.addAll(await _runMicrobench('lib/gestures/velocity_tracker_bench.dart')); + + return new TaskResult.success(allResults, benchmarkScoreKeys: allResults.keys.toList()); + }); +} + +Future _startFlutter({String command = 'run', List options: const [], bool canFail: false, Map env}) { + List args = ['run']..addAll(options); + return startProcess(path.join(flutterDirectory.path, 'bin', 'flutter'), args, env: env); +} + +Future> _readJsonResults(Process process) { + // IMPORTANT: keep these values in sync with dev/benchmarks/microbenchmarks/lib/common.dart + const String jsonStart = '================ RESULTS ================'; + const String jsonEnd = '================ FORMATTED =============='; + bool jsonStarted = false; + StringBuffer jsonBuf = new StringBuffer(); + Completer> completer = new Completer>(); + StreamSubscription stdoutSub; + + int prefixLength = 0; + stdoutSub = process.stdout + .transform(const Utf8Decoder()) + .transform(const LineSplitter()) + .listen((String line) { + print(line); + + if (line.contains(jsonStart)) { + jsonStarted = true; + prefixLength = line.indexOf(jsonStart); + return; + } + + if (line.contains(jsonEnd)) { + jsonStarted = false; + stdoutSub.cancel(); + process.kill(ProcessSignal.SIGINT); // flutter run doesn't quit automatically + completer.complete(JSON.decode(jsonBuf.toString())); + return; + } + + if (jsonStarted) + jsonBuf.writeln(line.substring(prefixLength)); + }); + + return completer.future; +} diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml index 9a29b91bf1..a55e113358 100644 --- a/dev/devicelab/manifest.yaml +++ b/dev/devicelab/manifest.yaml @@ -127,6 +127,12 @@ tasks: stage: devicelab required_agent_capabilities: ["has-android-device"] + microbenchmarks: + description: > + Runs benchmarks from dev/benchmarks/microbenchmarks. + stage: devicelab + required_agent_capabilities: ["has-android-device"] + # iOS on-device tests