From b81e361192fa908d9707b79bf2ffe69a6a186ca7 Mon Sep 17 00:00:00 2001 From: Ian Hickson Date: Mon, 6 Aug 2018 15:34:08 -0700 Subject: [PATCH] Revert "Refactor analysis benchmark and collect more data" (#20280) * Revert "increase size of user account drawer headers to 48 by 48 (#20266)" This reverts commit 4a7b4a4dde26af64e949388fe8e06d37337ed887. * Revert "EditableText Cursor can be set to not blink for testing (#20004)" This reverts commit d041b319e8bda5d02b4d83197eecd3f4ff8a97cf. * Revert "Refactor analysis benchmark and collect more data (#20169)" This reverts commit 5ea0a13598697df94937f691783a16b42535b47e. --- dev/devicelab/lib/tasks/analysis.dart | 135 +++++++++++--------------- 1 file changed, 55 insertions(+), 80 deletions(-) diff --git a/dev/devicelab/lib/tasks/analysis.dart b/dev/devicelab/lib/tasks/analysis.dart index deda4d7be5..73671a6636 100644 --- a/dev/devicelab/lib/tasks/analysis.dart +++ b/dev/devicelab/lib/tasks/analysis.dart @@ -4,20 +4,20 @@ import 'dart:async'; import 'dart:io'; +import 'dart:math' as math; import 'package:path/path.dart' as path; import '../framework/framework.dart'; import '../framework/utils.dart'; -/// Run each benchmark this many times and compute average, min, max. -/// -/// This must be small enough that we can do all the work in 15 minutes, the -/// devicelab deadline. Since there's four different analysis tasks, on average, -/// each can have 4 minutes. The tasks currently average a little more than a -/// minute, so that allows three runs per task. +/// Run each benchmark this many times and compute average. const int _kRunsPerBenchmark = 3; +/// Runs a benchmark once and reports the result as a lower-is-better numeric +/// value. +typedef Future _Benchmark(); + /// Path to the generated "mega gallery" app. Directory get _megaGalleryDirectory => dir(path.join(Directory.systemTemp.path, 'mega_gallery')); @@ -28,101 +28,76 @@ Future analyzerBenchmarkTask() async { await dart(['dev/tools/mega_gallery.dart', '--out=${_megaGalleryDirectory.path}']); }); - final Map data = {}; - data.addAll((await _run(new _FlutterRepoBenchmark())).asMap('flutter_repo', 'batch')); - data.addAll((await _run(new _FlutterRepoBenchmark(watch: true))).asMap('flutter_repo', 'watch')); - data.addAll((await _run(new _MegaGalleryBenchmark())).asMap('mega_gallery', 'batch')); - data.addAll((await _run(new _MegaGalleryBenchmark(watch: true))).asMap('mega_gallery', 'watch')); + final Map data = { + 'flutter_repo_batch': await _run(new _FlutterRepoBenchmark()), + 'flutter_repo_watch': await _run(new _FlutterRepoBenchmark(watch: true)), + 'mega_gallery_batch': await _run(new _MegaGalleryBenchmark()), + 'mega_gallery_watch': await _run(new _MegaGalleryBenchmark(watch: true)), + }; return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList()); } -class _BenchmarkResult { - const _BenchmarkResult(this.mean, this.min, this.max); - - final double mean; // seconds - - final double min; // seconds - - final double max; // seconds - - Map asMap(String benchmark, String mode) { - return { - '${benchmark}_$mode': mean, - '${benchmark}_${mode}_minimum': min, - '${benchmark}_${mode}_maximum': max, - }; - } -} - -abstract class _Benchmark { - _Benchmark({ this.watch = false }); +/// Times how long it takes to analyze the Flutter repository. +class _FlutterRepoBenchmark { + _FlutterRepoBenchmark({ this.watch = false }); final bool watch; - String get title; - - Directory get directory; - - List get options { - final List result = [ '--benchmark' ]; - if (watch) - options.add('--watch'); - return result; - } - - Future execute(int iteration, int targetIterations) async { - section('Analyze $title ${watch ? 'with watcher' : ''} - ${iteration + 1} / $targetIterations'); + Future call() async { + section('Analyze Flutter repo ${watch ? 'with watcher' : ''}'); final Stopwatch stopwatch = new Stopwatch(); - await inDirectory(directory, () async { + await inDirectory(flutterDirectory, () async { + final List options = [ + '--flutter-repo', + '--benchmark', + ]; + + if (watch) + options.add('--watch'); + stopwatch.start(); await flutter('analyze', options: options); stopwatch.stop(); }); - return stopwatch.elapsedMicroseconds / (1000.0 * 1000.0); - } -} - -/// Times how long it takes to analyze the Flutter repository. -class _FlutterRepoBenchmark extends _Benchmark { - _FlutterRepoBenchmark({ bool watch = false }) : super(watch: watch); - - @override - String get title => 'Flutter repo'; - - @override - Directory get directory => flutterDirectory; - - @override - List get options { - return super.options - ..add('--flutter-repo'); + return stopwatch.elapsedMilliseconds / 1000; } } /// Times how long it takes to analyze the generated "mega_gallery" app. -class _MegaGalleryBenchmark extends _Benchmark { - _MegaGalleryBenchmark({ bool watch = false }) : super(watch: watch); +class _MegaGalleryBenchmark { + _MegaGalleryBenchmark({ this.watch = false }); - @override - String get title => 'mega gallery'; + final bool watch; - @override - Directory get directory => _megaGalleryDirectory; + Future call() async { + section('Analyze mega gallery ${watch ? 'with watcher' : ''}'); + final Stopwatch stopwatch = new Stopwatch(); + await inDirectory(_megaGalleryDirectory, () async { + final List options = [ + '--benchmark', + ]; + + if (watch) + options.add('--watch'); + + stopwatch.start(); + await flutter('analyze', options: options); + stopwatch.stop(); + }); + return stopwatch.elapsedMilliseconds / 1000; + } } -/// Runs `benchmark` several times and reports the results. -Future<_BenchmarkResult> _run(_Benchmark benchmark) async { - final List results = []; - for (int i = 0; i < _kRunsPerBenchmark; i += 1) { +/// Runs a [benchmark] several times and reports the average result. +Future _run(_Benchmark benchmark) async { + double best; + for (int i = 0; i < _kRunsPerBenchmark; i++) { // Delete cached analysis results. rmTree(dir('${Platform.environment['HOME']}/.dartServer')); - results.add(await benchmark.execute(i, _kRunsPerBenchmark)); + + final double result = await benchmark(); + best = math.min(result, best ?? result); } - results.sort(); - final double sum = results.fold( - 0.0, - (double previousValue, double element) => previousValue + element, - ); - return new _BenchmarkResult(sum / results.length, results.first, results.last); + return best; }