diff --git a/dev/bots/run_command.dart b/dev/bots/run_command.dart index f39d89650b..7372f60b19 100644 --- a/dev/bots/run_command.dart +++ b/dev/bots/run_command.dart @@ -162,6 +162,15 @@ Future runCommand(String executable, List arguments, { }) async { final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}'; final String relativeWorkingDir = workingDirectory ?? path.relative(io.Directory.current.path); + if (dryRun) { + printProgress(_prettyPrintRunCommand(executable, arguments, workingDirectory)); + return CommandResult._( + 0, + Duration.zero, + '$executable ${arguments.join(' ')}', + 'Simulated execution due to --dry-run', + ); + } final Command command = await startCommand(executable, arguments, workingDirectory: workingDirectory, @@ -205,6 +214,23 @@ Future runCommand(String executable, List arguments, { return result; } +final String _flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(io.Platform.script)))); + +String _prettyPrintRunCommand(String executable, List arguments, String? workingDirectory) { + final StringBuffer output = StringBuffer(); + + // Print CWD relative to the root. + output.write('|> '); + output.write(path.relative(executable, from: _flutterRoot)); + if (workingDirectory != null) { + output.write(' (${path.relative(workingDirectory, from: _flutterRoot)})'); + } + output.writeln(': '); + output.writeAll(arguments.map((String a) => ' $a'), '\n'); + + return output.toString(); +} + /// Specifies what to do with the command output from [runCommand] and [startCommand]. enum OutputMode { /// Forwards standard output and standard error streams to the test process' diff --git a/dev/bots/suite_runners/run_web_tests.dart b/dev/bots/suite_runners/run_web_tests.dart index 4064c8c27a..d96c0467b2 100644 --- a/dev/bots/suite_runners/run_web_tests.dart +++ b/dev/bots/suite_runners/run_web_tests.dart @@ -708,7 +708,9 @@ class WebTestsSuite { // metriciFile is a transitional file that needs to be deleted once it is parsed. // TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting. // https://github.com/flutter/flutter/issues/146003 - metricFile.deleteSync(); + if (!dryRun) { + metricFile.deleteSync(); + } } // The `chromedriver` process created by this test. diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 0c9ff764d2..4077a937e1 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -4,14 +4,12 @@ // Runs the tests for the flutter/flutter repository. // -// // By default, test output is filtered and only errors are shown. (If a // particular test takes longer than _quietTimeout in utils.dart, the output is // shown then also, in case something has hung.) // // --verbose stops the output cleanup and just outputs everything verbatim. // -// // By default, errors are non-fatal; all tests are executed and the output // ends with a summary of the errors that were detected. // @@ -19,10 +17,10 @@ // // --abort-on-error causes the script to exit immediately when hitting an error. // -// // By default, all tests are run. However, the tests support being split by -// shard and subshard. (Inspect the code to see what shards and subshards are -// supported.) +// shard and subshard. Inspect the code to see what shards and subshards are +// supported, or run with `--dry-run` to get a list of tests that _would_ have +// been executed. // // If the CIRRUS_TASK_NAME environment variable exists, it is used to determine // the shard and sub-shard, by parsing it in the form shard-subshard-platform, @@ -43,7 +41,6 @@ // // --test-randomize-ordering-seed= sets the shuffle seed for reproducing runs. // -// // All other arguments are treated as arguments to pass to the flutter tool when // running tests. @@ -96,6 +93,7 @@ const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME'; Future main(List args) async { try { printProgress('STARTING ANALYSIS'); + bool dryRunArgSet = false; for (final String arg in args) { if (arg.startsWith('--local-engine=')) { localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length); @@ -116,10 +114,16 @@ Future main(List args) async { onError = () { system.exit(1); }; + } else if (arg == '--dry-run') { + dryRunArgSet = true; + printProgress('--dry-run enabled. Tests will not actually be executed.'); } else { flutterTestArgs.add(arg); } } + if (dryRunArgSet) { + enableDryRun(); + } if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) { printProgress('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}'); } @@ -541,6 +545,9 @@ Future _flutterBuild( } bool _allTargetsCached(File performanceFile) { + if (dryRun) { + return true; + } final Map data = json.decode(performanceFile.readAsStringSync()) as Map; final List> targets = (data['targets']! as List) diff --git a/dev/bots/test/test_test.dart b/dev/bots/test/test_test.dart index 2350e9bba4..49372b3f07 100644 --- a/dev/bots/test/test_test.dart +++ b/dev/bots/test/test_test.dart @@ -154,6 +154,15 @@ void main() { expectExitCode(result, 255); expect(result.stdout, contains('Invalid subshard name')); }); + + test('--dry-run prints every test that would run', () async { + final ProcessResult result = await runScript( + {}, + ['--dry-run'], + ); + expectExitCode(result, 0); + expect(result.stdout, contains('|> bin/flutter')); + }, testOn: 'posix'); }); test('selectTestsForSubShard distributes tests amongst subshards correctly', () async { diff --git a/dev/bots/utils.dart b/dev/bots/utils.dart index 3ed0958ff0..8b089e5839 100644 --- a/dev/bots/utils.dart +++ b/dev/bots/utils.dart @@ -68,9 +68,27 @@ const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME'; final Map localEngineEnv = {}; /// The arguments to pass to `flutter test` (typically the local engine -/// configuration) -- prefilled with the arguments passed to test.dart. +/// configuration) -- prefilled with the arguments passed to test.dart. final List flutterTestArgs = []; +/// Whether execution should be simulated for debugging purposes. +/// +/// When `true`, calls to [runCommand] print to [io.stdout] instead of running +/// the process. This is useful for determing what an invocation of `test.dart` +/// _might_ due if not invoked with `--dry-run`, or otherwise determine what the +/// different test shards and sub-shards are configured as. +bool get dryRun => _dryRun ?? false; + +/// Switches [dryRun] to `true`. +/// +/// Expected to be called at most once during execution of a process. +void enableDryRun() { + if (_dryRun != null) { + throw StateError('Should only be called at most once'); + } + _dryRun = true; +} +bool? _dryRun; const int kESC = 0x1B; const int kOpenSquareBracket = 0x5B; @@ -135,6 +153,10 @@ final List _pendingLogs = []; Timer? _hideTimer; // When this is null, the output is verbose. void foundError(List messages) { + if (dryRun) { + printProgress(messages.join('\n')); + return; + } assert(messages.isNotEmpty); // Make the error message easy to notice in the logs by // wrapping it in a red box. @@ -413,6 +435,10 @@ Future runDartTest(String workingDirectory, { removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null, ); + if (dryRun) { + return; + } + final TestFileReporterResults test = TestFileReporterResults.fromFile(metricFile); // --file-reporter name final File info = fileSystem.file(path.join(flutterRoot, 'error.log')); info.writeAsStringSync(json.encode(test.errors)); @@ -508,7 +534,9 @@ Future runFlutterTest(String workingDirectory, { // metriciFile is a transitional file that needs to be deleted once it is parsed. // TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting. // https://github.com/flutter/flutter/issues/146003 - metricFile.deleteSync(); + if (!dryRun) { + metricFile.deleteSync(); + } if (outputChecker != null) { final String? message = outputChecker(result); @@ -619,27 +647,33 @@ List selectIndexOfTotalSubshard(List tests, {String subshardKey = kSubs } Future _runFromList(Map items, String key, String name, int positionInTaskName) async { - String? item = Platform.environment[key]; - if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) { - final List parts = Platform.environment[CIRRUS_TASK_NAME]!.split('-'); - assert(positionInTaskName < parts.length); - item = parts[positionInTaskName]; - } - if (item == null) { - for (final String currentItem in items.keys) { - printProgress('$bold$key=$currentItem$reset'); - await items[currentItem]!(); + try { + String? item = Platform.environment[key]; + if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) { + final List parts = Platform.environment[CIRRUS_TASK_NAME]!.split('-'); + assert(positionInTaskName < parts.length); + item = parts[positionInTaskName]; } - } else { - printProgress('$bold$key=$item$reset'); - if (!items.containsKey(item)) { - foundError([ - '${red}Invalid $name: $item$reset', - 'The available ${name}s are: ${items.keys.join(", ")}', - ]); - return; + if (item == null) { + for (final String currentItem in items.keys) { + printProgress('$bold$key=$currentItem$reset'); + await items[currentItem]!(); + } + } else { + printProgress('$bold$key=$item$reset'); + if (!items.containsKey(item)) { + foundError([ + '${red}Invalid $name: $item$reset', + 'The available ${name}s are: ${items.keys.join(", ")}', + ]); + return; + } + await items[item]!(); + } + } catch (_) { + if (!dryRun) { + rethrow; } - await items[item]!(); } }