diff --git a/dev/devicelab/README.md b/dev/devicelab/README.md index 8655301957..59fd06c7ca 100644 --- a/dev/devicelab/README.md +++ b/dev/devicelab/README.md @@ -7,13 +7,15 @@ This package contains the code for test framework and the tests. More generally the tests are referred to as "tasks" in the API, but since we primarily use it for testing, this document refers to them as "tests". -If you have access to Google's internal network, you can see the continuous build results from the master branch at -. (There is currently no public view of this data, unfortunately.) +If you have access to Google's internal network, you can see the continuous +build results from the master branch at . +(There is currently no public view of this data, unfortunately.) # Prerequisites -You must set the `ANDROID_HOME` environment variable to run tests on Android. If you -have a local build of the Flutter engine, then you have a copy of the Android SDK at `.../engine/src/third_party/android_tools/sdk`. +You must set the `ANDROID_HOME` environment variable to run tests on Android. If +you have a local build of the Flutter engine, then you have a copy of the +Android SDK at `.../engine/src/third_party/android_tools/sdk`. # Running tests locally @@ -26,10 +28,16 @@ To run a test, use option `-t` (`--task`): ```sh # from the .../flutter/dev/devicelab directory -dart bin/run.dart -t {NAME_OF_TEST} +dart bin/run.dart -t {NAME_OR_PATH_OF_TEST} ``` -You can see the test names in the `manifest.yaml` file in this directory or by looking in `bin/tasks/`. Do not include the `.dart` file extension. For example, `dart bin/run.dart -t complex_layout__start_up`. +Where `NAME_OR_PATH_OF_TEST` can be either of: + +- the _name_ of a task, which you can find in the `manifest.yaml` file in this + directory. Example: `complex_layout__start_up`. +- the path to a Dart _file_ corresponding to a task, which resides in `bin/tasks`. + Tip: most shells support path auto-completion using the Tab key. Example: + `bin/tasks/complex_layout__start_up.dart`. To run multiple tests, repeat option `-t` (`--task`) multiple times: @@ -54,8 +62,8 @@ Currently there are only two stages defined, `devicelab` and `devicelab_ios`. # Reproducing broken builds locally If a commit caused a test to fail, -[the dashboard](http://go/flutter-dashboard/build.html) (requires access to the Google network, sorry) might look something -like this: +[the dashboard](http://go/flutter-dashboard/build.html) (requires access to the +Google network, sorry) might look something like this: ![Broken Test](images/broken-test.png) diff --git a/dev/devicelab/bin/run.dart b/dev/devicelab/bin/run.dart index a776a750e0..8ac3037edd 100644 --- a/dev/devicelab/bin/run.dart +++ b/dev/devicelab/bin/run.dart @@ -7,11 +7,14 @@ import 'dart:convert'; import 'dart:io'; import 'package:args/args.dart'; +import 'package:path/path.dart' as path; import 'package:flutter_devicelab/framework/manifest.dart'; import 'package:flutter_devicelab/framework/runner.dart'; import 'package:flutter_devicelab/framework/utils.dart'; +List _taskNames = []; + /// Runs tasks. /// /// The tasks are chosen depending on the command-line options @@ -28,24 +31,23 @@ Future main(List rawArgs) async { return null; } - List taskNames = []; - if (args.wasParsed('task')) { - taskNames.addAll(args['task']); - } else if (args.wasParsed('stage')) { - String stageName = args['stage']; - List tasks = loadTaskManifest().tasks; - for (ManifestTask task in tasks) { - if (task.stage == stageName) - taskNames.add(task.name); - } - } else if (args.wasParsed('all')) { - List tasks = loadTaskManifest().tasks; - for (ManifestTask task in tasks) { - taskNames.add(task.name); + if (!args.wasParsed('task')) { + if (args.wasParsed('stage')) { + String stageName = args['stage']; + List tasks = loadTaskManifest().tasks; + for (ManifestTask task in tasks) { + if (task.stage == stageName) + _taskNames.add(task.name); + } + } else if (args.wasParsed('all')) { + List tasks = loadTaskManifest().tasks; + for (ManifestTask task in tasks) { + _taskNames.add(task.name); + } } } - if (taskNames.isEmpty) { + if (_taskNames.isEmpty) { stderr.writeln('Failed to find tasks to run based on supplied options.'); exitCode = 1; return null; @@ -53,7 +55,7 @@ Future main(List rawArgs) async { bool silent = args['silent']; - for (String taskName in taskNames) { + for (String taskName in _taskNames) { section('Running task "$taskName"'); Map result = await runTask(taskName, silent: silent); @@ -73,9 +75,27 @@ final ArgParser _argParser = new ArgParser() abbr: 't', allowMultiple: true, splitCommas: true, - help: 'Name of the task to run. This option may be repeated to ' - 'specify multiple tasks. A task selected by name does not have to be ' - 'defined in manifest.yaml. It only needs a Dart executable in bin/tasks.', + help: 'Either:\n' + ' - the name of a task defined in manifest.yaml. Example: complex_layout__start_up.\n' + ' - the path to a Dart file corresponding to a task, which resides in bin/tasks. Example: bin/tasks/complex_layout__start_up.dart.\n' + '\n' + 'This option may be repeated to specify multiple tasks.', + callback: (List value) { + for (String nameOrPath in value) { + List fragments = path.split(nameOrPath); + bool isDartFile = fragments.last.endsWith('.dart'); + + if (fragments.length == 1 && !isDartFile) { + // Not a path + _taskNames.add(nameOrPath); + } else if (!isDartFile || fragments.length != 3 || !_listsEqual(['bin', 'tasks'], fragments.take(2).toList())) { + // Unsupported executable location + throw new FormatException('Invalid value for option -t (--task): $nameOrPath'); + } else { + _taskNames.add(path.withoutExtension(fragments.last)); + } + } + }, ) ..addOption( 'stage', @@ -106,3 +126,14 @@ final ArgParser _argParser = new ArgParser() negatable: true, defaultsTo: false, ); + +bool _listsEqual(List a, List b) { + if (a.length != b.length) return false; + + for (int i = 0; i < a.length; i++) { + if (a[i] != b[i]) + return false; + } + + return true; +} diff --git a/dev/devicelab/test/run_test.dart b/dev/devicelab/test/run_test.dart index 2a75476408..6c3ecda308 100644 --- a/dev/devicelab/test/run_test.dart +++ b/dev/devicelab/test/run_test.dart @@ -21,23 +21,31 @@ void main() { return scriptProcess.exitCode; } - test('Exits with code 0 when succeeds', () async { + test('exits with code 0 when succeeds', () async { expect(await runScript(['smoke_test_success']), 0); }); - test('Exits with code 1 when task throws', () async { + test('accepts file paths', () async { + expect(await runScript(['bin/tasks/smoke_test_success.dart']), 0); + }); + + test('rejects invalid file paths', () async { + expect(await runScript(['lib/framework/adb.dart']), 1); + }); + + test('exits with code 1 when task throws', () async { expect(await runScript(['smoke_test_throws']), 1); }); - test('Exits with code 1 when fails', () async { + test('exits with code 1 when fails', () async { expect(await runScript(['smoke_test_failure']), 1); }); - test('Exits with code 1 when fails to connect', () async { + test('exits with code 1 when fails to connect', () async { expect(await runScript(['smoke_test_setup_failure']), 1); }, skip: true); // https://github.com/flutter/flutter/issues/5901 - test('Exits with code 1 when results are mixed', () async { + test('exits with code 1 when results are mixed', () async { expect( await runScript([ 'smoke_test_failure',