diff --git a/dev/bots/prepare_package.dart b/dev/bots/prepare_package.dart index 72b73e09f0..ad3c7fa7ee 100644 --- a/dev/bots/prepare_package.dart +++ b/dev/bots/prepare_package.dart @@ -4,13 +4,14 @@ import 'dart:async'; import 'dart:convert'; -import 'dart:io'; +import 'dart:io' hide Platform; import 'dart:typed_data'; import 'package:args/args.dart'; import 'package:http/http.dart' as http; import 'package:path/path.dart' as path; import 'package:process/process.dart'; +import 'package:platform/platform.dart' show Platform, LocalPlatform; const String CHROMIUM_REPO = 'https://chromium.googlesource.com/external/github.com/flutter/flutter'; @@ -18,21 +19,157 @@ const String GITHUB_REPO = 'https://github.com/flutter/flutter.git'; const String MINGIT_FOR_WINDOWS_URL = 'https://storage.googleapis.com/flutter_infra/mingit/' '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip'; -/// Error class for when a process fails to run, so we can catch +/// Exception class for when a process fails to run, so we can catch /// it and provide something more readable than a stack trace. -class ProcessFailedException extends Error { - ProcessFailedException([this.message, this.exitCode]); +class ProcessRunnerException implements Exception { + ProcessRunnerException(this.message, [this.result]); - String message = ''; - int exitCode = 0; + final String message; + final ProcessResult result; + int get exitCode => result.exitCode ?? -1; @override - String toString() => message; + String toString() { + String output = runtimeType.toString(); + if (message != null) { + output += ': $message'; + } + final String stderr = result?.stderr ?? ''; + if (stderr.isNotEmpty) { + output += ':\n${result.stderr}'; + } + return output; + } +} + +enum Branch { dev, beta, release } + +String getBranchName(Branch branch) { + switch (branch) { + case Branch.beta: + return 'beta'; + case Branch.dev: + return 'dev'; + case Branch.release: + return 'release'; + } + return null; +} + +Branch fromBranchName(String name) { + switch (name) { + case 'beta': + return Branch.beta; + case 'dev': + return Branch.dev; + case 'release': + return Branch.release; + default: + throw new ArgumentError('Invalid branch name.'); + } +} + +/// A helper class for classes that want to run a process, optionally have the +/// stderr and stdout reported as the process runs, and capture the stdout +/// properly without dropping any. +class ProcessRunner { + ProcessRunner({ + this.processManager: const LocalProcessManager(), + this.subprocessOutput: true, + this.defaultWorkingDirectory, + this.platform: const LocalPlatform(), + }) { + environment = new Map.from(platform.environment); + } + + /// The platform to use for a starting environment. + final Platform platform; + + /// Set [subprocessOutput] to show output as processes run. Stdout from the + /// process will be printed to stdout, and stderr printed to stderr. + final bool subprocessOutput; + + /// Set the [processManager] in order to inject a test instance to perform + /// testing. + final ProcessManager processManager; + + /// Sets the default directory used when `workingDirectory` is not specified + /// to [runProcess]. + final Directory defaultWorkingDirectory; + + /// The environment to run processes with. + Map environment; + + /// Run the command and arguments in `commandLine` as a sub-process from + /// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses + /// [Directory.current] if [defaultWorkingDirectory] is not set. + /// + /// Set `failOk` if [runProcess] should not throw an exception when the + /// command completes with a a non-zero exit code. + Future runProcess( + List commandLine, { + Directory workingDirectory, + bool failOk: false, + }) async { + workingDirectory ??= defaultWorkingDirectory ?? Directory.current; + if (subprocessOutput) { + stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n'); + } + final List output = []; + final Completer stdoutComplete = new Completer(); + final Completer stderrComplete = new Completer(); + Process process; + Future allComplete() async { + await stderrComplete.future; + await stdoutComplete.future; + return process.exitCode; + } + + try { + process = await processManager.start( + commandLine, + workingDirectory: workingDirectory.absolute.path, + environment: environment, + ); + process.stdout.listen( + (List event) { + output.addAll(event); + if (subprocessOutput) { + stdout.add(event); + } + }, + onDone: () async => stdoutComplete.complete(), + ); + if (subprocessOutput) { + process.stderr.listen( + (List event) { + stderr.add(event); + }, + onDone: () async => stderrComplete.complete(), + ); + } else { + stderrComplete.complete(); + } + } on ProcessException catch (e) { + final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} ' + 'failed with:\n${e.toString()}'; + throw new ProcessRunnerException(message); + } + + final int exitCode = await allComplete(); + if (exitCode != 0 && !failOk) { + final String message = + 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed'; + throw new ProcessRunnerException( + message, new ProcessResult(0, exitCode, null, 'returned $exitCode')); + } + return UTF8.decoder.convert(output).trim(); + } } /// Creates a pre-populated Flutter archive from a git repo. class ArchiveCreator { - /// [_tempDir] is the directory to use for creating the archive. The script + /// [tempDir] is the directory to use for creating the archive. The script /// will place several GiB of data there, so it should have available space. /// /// The processManager argument is used to inject a mock of [ProcessManager] for @@ -40,52 +177,85 @@ class ArchiveCreator { /// /// If subprocessOutput is true, then output from processes invoked during /// archive creation is echoed to stderr and stdout. - ArchiveCreator(this._tempDir, {ProcessManager processManager, bool subprocessOutput: true}) - : _flutterRoot = new Directory(path.join(_tempDir.path, 'flutter')), - _processManager = processManager ?? const LocalProcessManager(), - _subprocessOutput = subprocessOutput { + ArchiveCreator( + this.tempDir, + this.outputDir, + this.revision, + this.branch, { + ProcessManager processManager, + bool subprocessOutput: true, + this.platform: const LocalPlatform(), + }) : assert(revision.length == 40), + flutterRoot = new Directory(path.join(tempDir.path, 'flutter')), + _processRunner = new ProcessRunner( + processManager: processManager, + subprocessOutput: subprocessOutput, + platform: platform, + ) { _flutter = path.join( - _flutterRoot.absolute.path, + flutterRoot.absolute.path, 'bin', 'flutter', ); - _environment = new Map.from(Platform.environment); - _environment['PUB_CACHE'] = path.join(_flutterRoot.absolute.path, '.pub-cache'); + _processRunner.environment['PUB_CACHE'] = path.join(flutterRoot.absolute.path, '.pub-cache'); } - final Directory _flutterRoot; - final Directory _tempDir; - final bool _subprocessOutput; - final ProcessManager _processManager; - String _flutter; + final Platform platform; + final Branch branch; + final String revision; + final Directory flutterRoot; + final Directory tempDir; + final Directory outputDir; final Uri _minGitUri = Uri.parse(MINGIT_FOR_WINDOWS_URL); - Map _environment; + final ProcessRunner _processRunner; + + File _outputFile; + String _version; + String _flutter; + + /// Get the name of the channel as a string. + String get branchName => getBranchName(branch); /// Returns a default archive name when given a Git revision. /// Used when an output filename is not given. - static String defaultArchiveName(String revision) { - final String os = Platform.operatingSystem.toLowerCase(); - final String id = revision.length > 10 ? revision.substring(0, 10) : revision; - final String suffix = Platform.isWindows ? 'zip' : 'tar.xz'; - return 'flutter_${os}_$id.$suffix'; + String get _archiveName { + final String os = platform.operatingSystem.toLowerCase(); + final String suffix = platform.isWindows ? 'zip' : 'tar.xz'; + return 'flutter_${os}_$_version-$branchName.$suffix'; + } + + /// Checks out the flutter repo and prepares it for other operations. + /// + /// Returns the version for this release, as obtained from the git tags. + Future initializeRepo() async { + await _checkoutFlutter(); + _version = await _getVersion(); + return _version; } /// Performs all of the steps needed to create an archive. - Future createArchive(String revision, File outputFile) async { - await _checkoutFlutter(revision); + Future createArchive() async { + assert(_version != null, 'Must run initializeRepo before createArchive'); + _outputFile = new File(path.join(outputDir.absolute.path, _archiveName)); await _installMinGitIfNeeded(); await _populateCaches(); - await _archiveFiles(outputFile); - return outputFile; + await _archiveFiles(_outputFile); + return _outputFile; + } + + /// Returns the version number of this release, according the to tags in + /// the repo. + Future _getVersion() async { + return _runGit(['describe', '--tags', '--abbrev=0']); } /// Clone the Flutter repo and make sure that the git environment is sane /// for when the user will unpack it. - Future _checkoutFlutter(String revision) async { - // We want the user to start out the in the 'master' branch instead of a - // detached head. To do that, we need to make sure master points at the + Future _checkoutFlutter() async { + // We want the user to start out the in the specified branch instead of a + // detached head. To do that, we need to make sure the branch points at the // desired revision. - await _runGit(['clone', '-b', 'master', CHROMIUM_REPO], workingDirectory: _tempDir); + await _runGit(['clone', '-b', branchName, CHROMIUM_REPO], workingDirectory: tempDir); await _runGit(['reset', '--hard', revision]); // Make the origin point to github instead of the chromium mirror. @@ -95,16 +265,17 @@ class ArchiveCreator { /// Retrieve the MinGit executable from storage and unpack it. Future _installMinGitIfNeeded() async { - if (!Platform.isWindows) { + if (!platform.isWindows) { return; } final Uint8List data = await http.readBytes(_minGitUri); - final File gitFile = new File(path.join(_tempDir.path, 'mingit.zip')); + final File gitFile = new File(path.join(tempDir.absolute.path, 'mingit.zip')); await gitFile.writeAsBytes(data, flush: true); - final Directory minGitPath = new Directory(path.join(_flutterRoot.path, 'bin', 'mingit')); + final Directory minGitPath = + new Directory(path.join(flutterRoot.absolute.path, 'bin', 'mingit')); await minGitPath.create(recursive: true); - await _unzipArchive(gitFile, currentDirectory: minGitPath); + await _unzipArchive(gitFile, workingDirectory: minGitPath); } /// Prepare the archive repo so that it has all of the caches warmed up and @@ -118,8 +289,11 @@ class ArchiveCreator { // Create each of the templates, since they will call 'pub get' on // themselves when created, and this will warm the cache with their // dependencies too. - for (String template in ['app', 'package', 'plugin']) { - final String createName = path.join(_tempDir.path, 'create_$template'); + // TODO(gspencer): 'package' is broken on dev branch right now! + // Add it back in once the following is fixed: + // https://github.com/flutter/flutter/issues/14448 + for (String template in ['app', 'plugin']) { + final String createName = path.join(tempDir.path, 'create_$template'); await _runFlutter( ['create', '--template=$template', createName], ); @@ -134,111 +308,154 @@ class ArchiveCreator { /// Write the archive to the given output file. Future _archiveFiles(File outputFile) async { if (outputFile.path.toLowerCase().endsWith('.zip')) { - await _createZipArchive(outputFile, _flutterRoot); + await _createZipArchive(outputFile, flutterRoot); } else if (outputFile.path.toLowerCase().endsWith('.tar.xz')) { - await _createTarArchive(outputFile, _flutterRoot); + await _createTarArchive(outputFile, flutterRoot); } } - Future _runFlutter(List args) => _runProcess([_flutter]..addAll(args)); + Future _runFlutter(List args, {Directory workingDirectory}) { + return _processRunner.runProcess([_flutter]..addAll(args), + workingDirectory: workingDirectory ?? flutterRoot); + } Future _runGit(List args, {Directory workingDirectory}) { - return _runProcess(['git']..addAll(args), workingDirectory: workingDirectory); + return _processRunner.runProcess(['git']..addAll(args), + workingDirectory: workingDirectory ?? flutterRoot); } /// Unpacks the given zip file into the currentDirectory (if set), or the /// same directory as the archive. /// /// May only be run on Windows (since 7Zip is not available on other platforms). - Future _unzipArchive(File archive, {Directory currentDirectory}) { - assert(Platform.isWindows); // 7Zip is only available on Windows. - currentDirectory ??= new Directory(path.dirname(archive.absolute.path)); + Future _unzipArchive(File archive, {Directory workingDirectory}) { + assert(platform.isWindows); // 7Zip is only available on Windows. + workingDirectory ??= new Directory(path.dirname(archive.absolute.path)); final List commandLine = ['7za', 'x', archive.absolute.path]; - return _runProcess(commandLine, workingDirectory: currentDirectory); + return _processRunner.runProcess(commandLine, workingDirectory: workingDirectory); } /// Create a zip archive from the directory source. /// /// May only be run on Windows (since 7Zip is not available on other platforms). Future _createZipArchive(File output, Directory source) { - assert(Platform.isWindows); // 7Zip is only available on Windows. + assert(platform.isWindows); // 7Zip is only available on Windows. final List commandLine = [ '7za', 'a', '-tzip', '-mx=9', output.absolute.path, - path.basename(source.absolute.path), + path.basename(source.path), ]; - return _runProcess(commandLine, + return _processRunner.runProcess(commandLine, workingDirectory: new Directory(path.dirname(source.absolute.path))); } /// Create a tar archive from the directory source. Future _createTarArchive(File output, Directory source) { - return _runProcess([ + return _processRunner.runProcess([ 'tar', 'cJf', output.absolute.path, path.basename(source.absolute.path), ], workingDirectory: new Directory(path.dirname(source.absolute.path))); } +} - /// Run the command and arguments in commandLine as a sub-process from - /// workingDirectory if set, or the current directory if not. - Future _runProcess(List commandLine, {Directory workingDirectory}) async { - workingDirectory ??= _flutterRoot; - if (_subprocessOutput) { - stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n'); - } - final List output = []; - final Completer stdoutComplete = new Completer(); - final Completer stderrComplete = new Completer(); - Process process; - Future allComplete() async { - await stderrComplete.future; - await stdoutComplete.future; - return process.exitCode; +class ArchivePublisher { + ArchivePublisher( + this.tempDir, + this.revision, + this.branch, + this.version, + this.outputFile, { + ProcessManager processManager, + bool subprocessOutput: true, + this.platform: const LocalPlatform(), + }) : assert(revision.length == 40), + platformName = platform.operatingSystem.toLowerCase(), + metadataGsPath = '$gsReleaseFolder/releases_${platform.operatingSystem.toLowerCase()}.json', + _processRunner = new ProcessRunner( + processManager: processManager, + subprocessOutput: subprocessOutput, + ); + + static String gsBase = 'gs://flutter_infra'; + static String releaseFolder = '/releases'; + static String gsReleaseFolder = '$gsBase$releaseFolder'; + static String baseUrl = 'https://storage.googleapis.com/flutter_infra'; + + final Platform platform; + final String platformName; + final String metadataGsPath; + final Branch branch; + final String revision; + final String version; + final Directory tempDir; + final File outputFile; + final ProcessRunner _processRunner; + String get branchName => getBranchName(branch); + String get destinationArchivePath => + '$branchName/$platformName/${path.basename(outputFile.path)}'; + + /// Publish the archive to Google Storage. + Future publishArchive() async { + final String destGsPath = '$gsReleaseFolder/$destinationArchivePath'; + await _cloudCopy(outputFile.absolute.path, destGsPath); + assert(tempDir.existsSync()); + return _updateMetadata(); + } + + Future _updateMetadata() async { + final String currentMetadata = await _runGsUtil(['cat', metadataGsPath]); + if (currentMetadata.isEmpty) { + throw new ProcessRunnerException('Empty metadata received from server'); } + Map jsonData; try { - process = await _processManager.start( - commandLine, - workingDirectory: workingDirectory.absolute.path, - environment: _environment, - ); - process.stdout.listen( - (List event) { - output.addAll(event); - if (_subprocessOutput) { - stdout.add(event); - } - }, - onDone: () async => stdoutComplete.complete(), - ); - if (_subprocessOutput) { - process.stderr.listen( - (List event) { - stderr.add(event); - }, - onDone: () async => stderrComplete.complete(), - ); - } else { - stderrComplete.complete(); - } - } on ProcessException catch (e) { - final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} ' - 'failed with:\n${e.toString()}'; - throw new ProcessFailedException(message, -1); + jsonData = json.decode(currentMetadata); + } on FormatException catch (e) { + throw new ProcessRunnerException('Unable to parse JSON metadata received from cloud: $e'); } - final int exitCode = await allComplete(); - if (exitCode != 0) { - final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} ' - 'failed with $exitCode.'; - throw new ProcessFailedException(message, exitCode); + // Update the metadata file with the data for this package. + jsonData['base_url'] = '$baseUrl$releaseFolder'; + if (!jsonData.containsKey('current_release')) { + jsonData['current_release'] = {}; } - return UTF8.decoder.convert(output).trim(); + jsonData['current_release'][branchName] = revision; + if (!jsonData.containsKey('releases')) { + jsonData['releases'] = {}; + } + if (!jsonData['releases'].containsKey(revision)) { + jsonData['releases'][revision] = >{}; + } + final Map metadata = {}; + metadata['${platformName}_archive'] = destinationArchivePath; + metadata['release_date'] = new DateTime.now().toUtc().toIso8601String(); + metadata['version'] = version; + jsonData['releases'][revision][branchName] = metadata; + + final File tempFile = new File(path.join(tempDir.absolute.path, 'releases_$platformName.json')); + final JsonEncoder encoder = const JsonEncoder.withIndent(' '); + tempFile.writeAsStringSync(encoder.convert(jsonData)); + await _cloudCopy(tempFile.absolute.path, metadataGsPath); + } + + Future _runGsUtil(List args, + {Directory workingDirectory, bool failOk: false}) async { + return _processRunner.runProcess( + ['gsutil']..addAll(args), + workingDirectory: workingDirectory, + failOk: failOk, + ); + } + + Future _cloudCopy(String src, String dest) async { + await _runGsUtil(['rm', dest], failOk: true); + return _runGsUtil(['cp', src, dest]); } } @@ -256,23 +473,37 @@ Future main(List argList) async { defaultsTo: null, help: 'A location where temporary files may be written. Defaults to a ' 'directory in the system temp folder. Will write a few GiB of data, ' - 'so it should have sufficient free space.', + 'so it should have sufficient free space. If a temp_dir is not ' + 'specified, then the default temp_dir will be created, used, and ' + 'removed automatically.', ); + argParser.addOption('revision', + defaultsTo: null, + help: 'The Flutter git repo revision to build the ' + 'archive with. Must be the full 40-character hash. Required.'); argParser.addOption( - 'revision', - defaultsTo: 'master', - help: 'The Flutter revision to build the archive with. Defaults to the ' - "master branch's HEAD revision.", + 'branch', + defaultsTo: null, + allowed: Branch.values.map((Branch branch) => getBranchName(branch)), + help: 'The Flutter branch to build the archive with. Required.', ); argParser.addOption( 'output', defaultsTo: null, - help: 'The path to the file where the output archive should be ' - 'written. The output file must end in ".tar.xz" on Linux and Mac, ' - 'and ".zip" on Windows. If --output is not specified, the archive will ' - "be written to the current directory. If the output directory doesn't " - 'exist, it, and the path to it, will be created.', + help: 'The path to the directory where the output archive should be ' + 'written. If --output is not specified, the archive will be written to ' + "the current directory. If the output directory doesn't exist, it, and " + 'the path to it, will be created.', ); + argParser.addFlag( + 'publish', + defaultsTo: false, + help: 'The path to the directory where the output archive should be ' + 'written. If --output is not specified, the archive will be written to ' + "the current directory. If the output directory doesn't exist, it, and " + 'the path to it, will be created.', + ); + final ArgResults args = argParser.parse(argList); void errorExit(String message, {int exitCode = -1}) { @@ -281,9 +512,17 @@ Future main(List argList) async { exit(exitCode); } - if (args['revision'].isEmpty) { + final String revision = args['revision']; + if (revision.isEmpty) { errorExit('Invalid argument: --revision must be specified.'); } + if (revision.length != 40) { + errorExit('Invalid argument: --revision must be the entire hash, not just a prefix.'); + } + + if (args['branch'].isEmpty) { + errorExit('Invalid argument: --branch must be specified.'); + } Directory tempDir; bool removeTempDir = false; @@ -297,34 +536,34 @@ Future main(List argList) async { } } - final String output = (args['output'] == null || args['output'].isEmpty) - ? path.join(path.current, ArchiveCreator.defaultArchiveName(args['revision'])) - : args['output']; - - /// Sanity check the output filename. - final String outputFilename = path.basename(output); - if (Platform.isWindows) { - if (!outputFilename.endsWith('.zip')) { - errorExit('The argument to --output must end in .zip on Windows.'); - } + Directory outputDir; + if (args['output'] == null) { + outputDir = tempDir; } else { - if (!outputFilename.endsWith('.tar.xz')) { - errorExit('The argument to --output must end in .tar.xz on Linux and Mac.'); + outputDir = new Directory(args['output']); + if (!outputDir.existsSync()) { + outputDir.createSync(recursive: true); } } - final Directory outputDirectory = new Directory(path.dirname(output)); - if (!outputDirectory.existsSync()) { - outputDirectory.createSync(recursive: true); - } - final File outputFile = new File(path.join(outputDirectory.absolute.path, outputFilename)); - - final ArchiveCreator preparer = new ArchiveCreator(tempDir); + final Branch branch = fromBranchName(args['branch']); + final ArchiveCreator creator = new ArchiveCreator(tempDir, outputDir, revision, branch); int exitCode = 0; String message; try { - await preparer.createArchive(args['revision'], outputFile); - } on ProcessFailedException catch (e) { + final String version = await creator.initializeRepo(); + final File outputFile = await creator.createArchive(); + if (args['publish']) { + final ArchivePublisher publisher = new ArchivePublisher( + tempDir, + revision, + branch, + version, + outputFile, + ); + await publisher.publishArchive(); + } + } on ProcessRunnerException catch (e) { exitCode = e.exitCode; message = e.message; } finally { diff --git a/dev/tools/test/fake_process_manager.dart b/dev/bots/test/fake_process_manager.dart similarity index 69% rename from dev/tools/test/fake_process_manager.dart rename to dev/bots/test/fake_process_manager.dart index 7c4c456b4f..54471d9124 100644 --- a/dev/tools/test/fake_process_manager.dart +++ b/dev/bots/test/fake_process_manager.dart @@ -29,111 +29,88 @@ class FakeProcessManager extends Mock implements ProcessManager { /// The list of results that will be sent back, organized by the command line /// that will produce them. Each command line has a list of returned stdout /// output that will be returned on each successive call. - Map> fakeResults = >{}; + Map> _fakeResults = >{}; + Map> get fakeResults => _fakeResults; + set fakeResults(Map> value) { + _fakeResults = >{}; + for (String key in value.keys) { + _fakeResults[key] = [] + ..addAll(value[key] ?? [new ProcessResult(0, 0, '', '')]); + } + } /// The list of invocations that occurred, in the order they occurred. List invocations = []; - /// Verify that the given command lines were called, in the given order. + /// Verify that the given command lines were called, in the given order, and that the + /// parameters were in the same order. void verifyCalls(List calls) { int index = 0; - expect(invocations.length, equals(calls.length)); for (String call in calls) { expect(call.split(' '), orderedEquals(invocations[index].positionalArguments[0])); index++; } + expect(invocations.length, equals(calls.length)); } - /// Sets the list of results that will be returned from each successive call. - void setResults(Map> results) { - final Map> resultCodeUnits = >{}; - for (String key in results.keys) { - resultCodeUnits[key] = - results[key].map((String result) => new ProcessResult(0, 0, result, '')).toList(); - } - fakeResults = resultCodeUnits; - } - - ProcessResult _popResult(String key) { + ProcessResult _popResult(List command) { + final String key = command.join(' '); expect(fakeResults, isNotEmpty); expect(fakeResults, contains(key)); expect(fakeResults[key], isNotEmpty); return fakeResults[key].removeAt(0); } - FakeProcess _popProcess(String key) => - new FakeProcess(_popResult(key), stdinResults: stdinResults); + FakeProcess _popProcess(List command) => + new FakeProcess(_popResult(command), stdinResults: stdinResults); Future _nextProcess(Invocation invocation) async { invocations.add(invocation); - return new Future.value(_popProcess(invocation.positionalArguments[0].join(' '))); + return new Future.value(_popProcess(invocation.positionalArguments[0])); } ProcessResult _nextResultSync(Invocation invocation) { invocations.add(invocation); - return _popResult(invocation.positionalArguments[0].join(' ')); + return _popResult(invocation.positionalArguments[0]); } Future _nextResult(Invocation invocation) async { invocations.add(invocation); - return new Future.value(_popResult(invocation.positionalArguments[0].join(' '))); + return new Future.value(_popResult(invocation.positionalArguments[0])); } void _setupMock() { // Note that not all possible types of invocations are covered here, just the ones // expected to be called. // TODO(gspencer): make this more general so that any call will be captured. - when( - start( - typed(captureAny), - environment: typed(captureAny, named: 'environment'), - workingDirectory: typed(captureAny, named: 'workingDirectory'), - ), - ).thenAnswer(_nextProcess); + when(start( + typed(captureAny), + environment: typed(captureAny, named: 'environment'), + workingDirectory: typed(captureAny, named: 'workingDirectory'), + )).thenAnswer(_nextProcess); - when( - start( - typed(captureAny), - ), - ).thenAnswer(_nextProcess); + when(start(typed(captureAny))).thenAnswer(_nextProcess); - when( - run( - typed(captureAny), - environment: typed(captureAny, named: 'environment'), - workingDirectory: typed(captureAny, named: 'workingDirectory'), - ), - ).thenAnswer(_nextResult); + when(run( + typed(captureAny), + environment: typed(captureAny, named: 'environment'), + workingDirectory: typed(captureAny, named: 'workingDirectory'), + )).thenAnswer(_nextResult); - when( - run( - typed(captureAny), - ), - ).thenAnswer(_nextResult); + when(run(typed(captureAny))).thenAnswer(_nextResult); - when( - runSync( - typed(captureAny), - environment: typed(captureAny, named: 'environment'), - workingDirectory: typed(captureAny, named: 'workingDirectory'), - ), - ).thenAnswer(_nextResultSync); + when(runSync( + typed(captureAny), + environment: typed(captureAny, named: 'environment'), + workingDirectory: typed(captureAny, named: 'workingDirectory') + )).thenAnswer(_nextResultSync); - when( - runSync( - typed(captureAny), - ), - ).thenAnswer(_nextResultSync); + when(runSync(typed(captureAny))).thenAnswer(_nextResultSync); when(killPid(typed(captureAny), typed(captureAny))).thenReturn(true); - when( - canRun(captureAny, - workingDirectory: typed( - captureAny, - named: 'workingDirectory', - )), - ).thenReturn(true); + when(canRun(captureAny, workingDirectory: typed(captureAny, named: 'workingDirectory'))) + .thenReturn(true); } } @@ -190,9 +167,11 @@ class StringStreamConsumer implements StreamConsumer> { Future addStream(Stream> value) { streams.add(value); completers.add(new Completer()); - subscriptions.add(value.listen((List data) { - sendString(utf8.decode(data)); - })); + subscriptions.add( + value.listen((List data) { + sendString(utf8.decode(data)); + }), + ); subscriptions.last.onDone(() => completers.last.complete(null)); return new Future.value(null); } diff --git a/dev/tools/test/fake_process_manager_test.dart b/dev/bots/test/fake_process_manager_test.dart similarity index 57% rename from dev/tools/test/fake_process_manager_test.dart rename to dev/bots/test/fake_process_manager_test.dart index ee6a97c82d..d94f6efac5 100644 --- a/dev/tools/test/fake_process_manager_test.dart +++ b/dev/bots/test/fake_process_manager_test.dart @@ -25,11 +25,15 @@ void main() { tearDown(() async {}); test('start works', () async { - final Map> calls = >{ - 'gsutil acl get gs://flutter_infra/releases/releases.json': ['output1'], - 'gsutil cat gs://flutter_infra/releases/releases.json': ['test'], + final Map> calls = >{ + 'gsutil acl get gs://flutter_infra/releases/releases.json': [ + new ProcessResult(0, 0, 'output1', '') + ], + 'gsutil cat gs://flutter_infra/releases/releases.json': [ + new ProcessResult(0, 0, 'output2', '') + ], }; - processManager.setResults(calls); + processManager.fakeResults = calls; for (String key in calls.keys) { final Process process = await processManager.start(key.split(' ')); String output = ''; @@ -37,56 +41,68 @@ void main() { output += utf8.decode(item); }); await process.exitCode; - expect(output, equals(calls[key][0])); + expect(output, equals(calls[key][0].stdout)); } - processManager.verifyCalls(calls.keys); + processManager.verifyCalls(calls.keys.toList()); }); test('run works', () async { - final Map> calls = >{ - 'gsutil acl get gs://flutter_infra/releases/releases.json': ['output1'], - 'gsutil cat gs://flutter_infra/releases/releases.json': ['test'], + final Map> calls = >{ + 'gsutil acl get gs://flutter_infra/releases/releases.json': [ + new ProcessResult(0, 0, 'output1', '') + ], + 'gsutil cat gs://flutter_infra/releases/releases.json': [ + new ProcessResult(0, 0, 'output2', '') + ], }; - processManager.setResults(calls); + processManager.fakeResults = calls; for (String key in calls.keys) { final ProcessResult result = await processManager.run(key.split(' ')); - expect(result.stdout, equals(calls[key][0])); + expect(result.stdout, equals(calls[key][0].stdout)); } - processManager.verifyCalls(calls.keys); + processManager.verifyCalls(calls.keys.toList()); }); test('runSync works', () async { - final Map> calls = >{ - 'gsutil acl get gs://flutter_infra/releases/releases.json': ['output1'], - 'gsutil cat gs://flutter_infra/releases/releases.json': ['test'], + final Map> calls = >{ + 'gsutil acl get gs://flutter_infra/releases/releases.json': [ + new ProcessResult(0, 0, 'output1', '') + ], + 'gsutil cat gs://flutter_infra/releases/releases.json': [ + new ProcessResult(0, 0, 'output2', '') + ], }; - processManager.setResults(calls); + processManager.fakeResults = calls; for (String key in calls.keys) { final ProcessResult result = processManager.runSync(key.split(' ')); - expect(result.stdout, equals(calls[key][0])); + expect(result.stdout, equals(calls[key][0].stdout)); } - processManager.verifyCalls(calls.keys); + processManager.verifyCalls(calls.keys.toList()); }); test('captures stdin', () async { - final Map> calls = >{ - 'gsutil acl get gs://flutter_infra/releases/releases.json': ['output1'], - 'gsutil cat gs://flutter_infra/releases/releases.json': ['test'], + final Map> calls = >{ + 'gsutil acl get gs://flutter_infra/releases/releases.json': [ + new ProcessResult(0, 0, 'output1', '') + ], + 'gsutil cat gs://flutter_infra/releases/releases.json': [ + new ProcessResult(0, 0, 'output2', '') + ], }; - processManager.setResults(calls); + processManager.fakeResults = calls; for (String key in calls.keys) { final Process process = await processManager.start(key.split(' ')); String output = ''; process.stdout.listen((List item) { output += utf8.decode(item); }); - final String testInput = '${calls[key][0]} input'; + final String testInput = '${calls[key][0].stdout} input'; process.stdin.add(testInput.codeUnits); await process.exitCode; - expect(output, equals(calls[key][0])); + expect(output, equals(calls[key][0].stdout)); expect(stdinCaptured.last, equals(testInput)); } - processManager.verifyCalls(calls.keys); + processManager.verifyCalls(calls.keys.toList()); }); }); } diff --git a/dev/bots/test/prepare_package_test.dart b/dev/bots/test/prepare_package_test.dart index 7b9b68d251..c517ae658a 100644 --- a/dev/bots/test/prepare_package_test.dart +++ b/dev/bots/test/prepare_package_test.dart @@ -2,154 +2,247 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:async'; -import 'dart:io'; +import 'dart:convert'; +import 'dart:io' hide Platform; import 'package:mockito/mockito.dart'; import 'package:test/test.dart'; import 'package:path/path.dart' as path; -import 'package:process/process.dart'; +import 'package:platform/platform.dart' show FakePlatform; import '../prepare_package.dart'; +import 'fake_process_manager.dart'; void main() { - group('ArchiveCreator', () { - ArchiveCreator preparer; - Directory tmpDir; - Directory flutterDir; - File outputFile; - MockProcessManager processManager; - List results = []; - final List> args = >[]; - final List> namedArgs = >[]; - String flutterExe; + final String testRef = 'deadbeefdeadbeefdeadbeefdeadbeefdeadbeef'; + for (String platformName in ['macos', 'linux', 'windows']) { + final FakePlatform platform = new FakePlatform( + operatingSystem: platformName, + environment: {}, + ); + group('ArchiveCreator for $platformName', () { + ArchiveCreator creator; + Directory tmpDir; + Directory flutterDir; + FakeProcessManager processManager; + final List> args = >[]; + final List> namedArgs = >[]; + String flutter; - void _verifyCommand(List args, String expected) { - final List expectedList = expected.split(' '); - expect(args[0], orderedEquals(expectedList)); + setUp(() async { + processManager = new FakeProcessManager(); + args.clear(); + namedArgs.clear(); + tmpDir = await Directory.systemTemp.createTemp('flutter_'); + flutterDir = new Directory(path.join(tmpDir.path, 'flutter')); + flutterDir.createSync(recursive: true); + creator = new ArchiveCreator( + tmpDir, + tmpDir, + testRef, + Branch.dev, + processManager: processManager, + subprocessOutput: false, + platform: platform, + ); + flutter = path.join(creator.flutterRoot.absolute.path, 'bin', 'flutter'); + }); + + tearDown(() async { + // On Windows, the directory is locked and not able to be deleted yet. So + // we just leave some (very small, because we're not actually building + // archives here) trash around to be deleted at the next reboot. + if (!platform.isWindows) { + await tmpDir.delete(recursive: true); + } + }); + + test('sets PUB_CACHE properly', () async { + final String createBase = path.join(tmpDir.absolute.path, 'create_'); + final Map> calls = >{ + 'git clone -b dev https://chromium.googlesource.com/external/github.com/flutter/flutter': + null, + 'git reset --hard $testRef': null, + 'git remote remove origin': null, + 'git remote add origin https://github.com/flutter/flutter.git': null, + 'git describe --tags --abbrev=0': [new ProcessResult(0, 0, 'v1.2.3', '')], + }; + if (platform.isWindows) { + calls['7za x ${path.join(tmpDir.path, 'mingit.zip')}'] = null; + } + calls.addAll(>{ + '$flutter doctor': null, + '$flutter update-packages': null, + '$flutter precache': null, + '$flutter ide-config': null, + '$flutter create --template=app ${createBase}app': null, + '$flutter create --template=package ${createBase}package': null, + '$flutter create --template=plugin ${createBase}plugin': null, + 'git clean -f -X **/.packages': null, + }); + final String archiveName = path.join(tmpDir.absolute.path, + 'flutter_${platformName}_v1.2.3-dev${platform.isWindows ? '.zip' : '.tar.xz'}'); + if (platform.isWindows) { + calls['7za a -tzip -mx=9 $archiveName flutter'] = null; + } else { + calls['tar cJf $archiveName flutter'] = null; + } + processManager.fakeResults = calls; + await creator.initializeRepo(); + await creator.createArchive(); + expect( + verify(processManager.start( + captureAny, + workingDirectory: captureAny, + environment: captureAny, + )).captured[1]['PUB_CACHE'], + endsWith(path.join('flutter', '.pub-cache')), + ); + }); + + test('calls the right commands for archive output', () async { + final String createBase = path.join(tmpDir.absolute.path, 'create_'); + final Map> calls = >{ + 'git clone -b dev https://chromium.googlesource.com/external/github.com/flutter/flutter': + null, + 'git reset --hard $testRef': null, + 'git remote remove origin': null, + 'git remote add origin https://github.com/flutter/flutter.git': null, + 'git describe --tags --abbrev=0': [new ProcessResult(0, 0, 'v1.2.3', '')], + }; + if (platform.isWindows) { + calls['7za x ${path.join(tmpDir.path, 'mingit.zip')}'] = null; + } + calls.addAll(>{ + '$flutter doctor': null, + '$flutter update-packages': null, + '$flutter precache': null, + '$flutter ide-config': null, + '$flutter create --template=app ${createBase}app': null, + // TODO(gspencer): Re-enable this when package works again: + // https://github.com/flutter/flutter/issues/14448 + // '$flutter create --template=package ${createBase}package': null, + '$flutter create --template=plugin ${createBase}plugin': null, + 'git clean -f -X **/.packages': null, + }); + final String archiveName = path.join(tmpDir.absolute.path, + 'flutter_${platformName}_v1.2.3-dev${platform.isWindows ? '.zip' : '.tar.xz'}'); + if (platform.isWindows) { + calls['7za a -tzip -mx=9 $archiveName flutter'] = null; + } else { + calls['tar cJf $archiveName flutter'] = null; + } + processManager.fakeResults = calls; + creator = new ArchiveCreator( + tmpDir, + tmpDir, + testRef, + Branch.dev, + processManager: processManager, + subprocessOutput: false, + platform: platform, + ); + await creator.initializeRepo(); + await creator.createArchive(); + processManager.verifyCalls(calls.keys.toList()); + }); + + test('throws when a command errors out', () async { + final Map> calls = >{ + 'git clone -b dev https://chromium.googlesource.com/external/github.com/flutter/flutter': + [new ProcessResult(0, 0, 'output1', '')], + 'git reset --hard $testRef': [new ProcessResult(0, -1, 'output2', '')], + }; + processManager.fakeResults = calls; + expect(expectAsync0(creator.initializeRepo), + throwsA(const isInstanceOf())); + }); + }); + + group('ArchivePublisher for $platformName', () { + FakeProcessManager processManager; + Directory tempDir; + + setUp(() async { + processManager = new FakeProcessManager(); + tempDir = await Directory.systemTemp.createTemp('flutter_'); + tempDir.createSync(); + }); + + tearDown(() async { + // On Windows, the directory is locked and not able to be deleted yet. So + // we just leave some (very small, because we're not actually building + // archives here) trash around to be deleted at the next reboot. + if (!platform.isWindows) { + await tempDir.delete(recursive: true); + } + }); + + test('calls the right processes', () async { + final String releasesName = 'releases_$platformName.json'; + final String archivePath = path.join(tempDir.absolute.path, 'output_archive'); + final String gsArchivePath = 'gs://flutter_infra/releases/dev/$platformName/output_archive'; + final String jsonPath = path.join(tempDir.absolute.path, releasesName); + final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName'; + final String releasesJson = '''{ + "base_url": "https://storage.googleapis.com/flutter_infra/releases", + "current_release": { + "beta": "6da8ec6bd0c4801b80d666869e4069698561c043", + "dev": "f88c60b38c3a5ef92115d24e3da4175b4890daba" + }, + "releases": { + "6da8ec6bd0c4801b80d666869e4069698561c043": { + "${platformName}_archive": "dev/linux/flutter_${platformName}_0.21.0-beta.tar.xz", + "release_date": "2017-12-19T10:30:00,847287019-08:00", + "version": "0.21.0-beta" + }, + "f88c60b38c3a5ef92115d24e3da4175b4890daba": { + "${platformName}_archive": "dev/linux/flutter_${platformName}_0.22.0-dev.tar.xz", + "release_date": "2018-01-19T13:30:09,728487019-08:00", + "version": "0.22.0-dev" + } } - - Future _nextResult(Invocation invocation) async { - args.add(invocation.positionalArguments); - namedArgs.add(invocation.namedArguments); - final Process result = results.isEmpty ? new MockProcess('', '', 0) : results.removeAt(0); - return new Future.value(result); - } - - void _answerWithResults() { - when( - processManager.start( - typed(captureAny), - environment: typed(captureAny, named: 'environment'), - workingDirectory: typed(captureAny, named: 'workingDirectory'), - ), - ).thenAnswer(_nextResult); - } - - setUp(() async { - processManager = new MockProcessManager(); - args.clear(); - namedArgs.clear(); - tmpDir = await Directory.systemTemp.createTemp('flutter_'); - outputFile = - new File(path.join(tmpDir.absolute.path, ArchiveCreator.defaultArchiveName('master'))); - flutterDir = new Directory(path.join(tmpDir.path, 'flutter')); - flutterDir.createSync(recursive: true); - flutterExe = - path.join(flutterDir.path, 'bin', 'flutter'); - }); - - tearDown(() async { - // On Windows, the directory is locked and not able to be deleted, because it is a - // temporary directory. So we just leave some (very small, because we're not actually - // building archives here) trash around to be deleted at the next reboot. - if (!Platform.isWindows) { - await tmpDir.delete(recursive: true); - } - }); - - test('sets PUB_CACHE properly', () async { - preparer = - new ArchiveCreator(tmpDir, processManager: processManager, subprocessOutput: false); - _answerWithResults(); - await preparer.createArchive('master', outputFile); - expect( - verify(processManager.start( - captureAny, - workingDirectory: captureAny, - environment: captureAny, - )).captured[1]['PUB_CACHE'], - endsWith(path.join('flutter', '.pub-cache')), - ); - }); - - test('calls the right commands for archive output', () async { - preparer = - new ArchiveCreator(tmpDir, processManager: processManager, subprocessOutput: false); - _answerWithResults(); - await preparer.createArchive('master', outputFile); - final List commands = [ - 'git clone -b master https://chromium.googlesource.com/external/github.com/flutter/flutter', - 'git reset --hard master', - 'git remote remove origin', - 'git remote add origin https://github.com/flutter/flutter.git', - ]; - if (Platform.isWindows) { - commands.add('7za x ${path.join(tmpDir.path, 'mingit.zip')}'); - } - commands.addAll([ - '$flutterExe doctor', - '$flutterExe update-packages', - '$flutterExe precache', - '$flutterExe ide-config', - '$flutterExe create --template=app ${path.join(tmpDir.path, 'create_app')}', - '$flutterExe create --template=package ${path.join(tmpDir.path, 'create_package')}', - '$flutterExe create --template=plugin ${path.join(tmpDir.path, 'create_plugin')}', - 'git clean -f -X **/.packages', - ]); - if (Platform.isWindows) { - commands.add('7za a -tzip -mx=9 ${outputFile.absolute.path} flutter'); - } else { - commands.add('tar cJf ${outputFile.absolute.path} flutter'); - } - int step = 0; - for (String command in commands) { - _verifyCommand(args[step++], command); - } - }); - - test('throws when a command errors out', () async { - preparer = - new ArchiveCreator(tmpDir, processManager: processManager, subprocessOutput: false); - - results = [ - new MockProcess('', '', 0), - new MockProcess('', "Don't panic.\n", -1) - ]; - _answerWithResults(); - expect(expectAsync2(preparer.createArchive)('master', new File('foo')), - throwsA(const isInstanceOf())); - }); - }); } - -class MockProcessManager extends Mock implements ProcessManager {} - -class MockProcess extends Mock implements Process { - MockProcess(this._stdout, [this._stderr, this._exitCode]); - - String _stdout; - String _stderr; - int _exitCode; - - @override - Stream> get stdout => - new Stream>.fromIterable(>[_stdout.codeUnits]); - - @override - Stream> get stderr => - new Stream>.fromIterable(>[_stderr.codeUnits]); - - @override - Future get exitCode => new Future.value(_exitCode); +'''; + final Map> calls = >{ + 'gsutil rm $gsArchivePath': null, + 'gsutil cp $archivePath $gsArchivePath': null, + 'gsutil cat $gsJsonPath': [new ProcessResult(0, 0, releasesJson, '')], + 'gsutil rm $gsJsonPath': null, + 'gsutil cp $jsonPath $gsJsonPath': null, + }; + processManager.fakeResults = calls; + final File outputFile = new File(path.join(tempDir.absolute.path, 'output_archive')); + assert(tempDir.existsSync()); + final ArchivePublisher publisher = new ArchivePublisher( + tempDir, + testRef, + Branch.dev, + '1.2.3', + outputFile, + processManager: processManager, + subprocessOutput: false, + platform: platform, + ); + assert(tempDir.existsSync()); + await publisher.publishArchive(); + processManager.verifyCalls(calls.keys.toList()); + final File releaseFile = new File(jsonPath); + expect(releaseFile.existsSync(), isTrue); + final String contents = releaseFile.readAsStringSync(); + // Make sure new data is added. + expect(contents, contains('"dev": "$testRef"')); + expect(contents, contains('"$testRef": {')); + expect(contents, contains('"${platformName}_archive": "dev/$platformName/output_archive"')); + // Make sure existing entries are preserved. + expect(contents, contains('"6da8ec6bd0c4801b80d666869e4069698561c043": {')); + expect(contents, contains('"f88c60b38c3a5ef92115d24e3da4175b4890daba": {')); + expect(contents, contains('"beta": "6da8ec6bd0c4801b80d666869e4069698561c043"')); + // Make sure it's valid JSON, and in the right format. + final Map jsonData = json.decode(contents); + final JsonEncoder encoder = const JsonEncoder.withIndent(' '); + expect(contents, equals(encoder.convert(jsonData))); + }); + }); + } } diff --git a/dev/tools/lib/archive_publisher.dart b/dev/tools/lib/archive_publisher.dart deleted file mode 100644 index afad198cb1..0000000000 --- a/dev/tools/lib/archive_publisher.dart +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:convert'; -import 'dart:io'; - -import 'package:path/path.dart' as path; -import 'package:process/process.dart'; - -class ArchivePublisherException implements Exception { - ArchivePublisherException(this.message, [this.result]); - - final String message; - final ProcessResult result; - - @override - String toString() { - String output = 'ArchivePublisherException'; - if (message != null) { - output += ': $message'; - } - final String stderr = result?.stderr ?? ''; - if (stderr.isNotEmpty) { - output += ':\n${result.stderr}'; - } - return output; - } -} - -enum Channel { dev, beta } - -/// Publishes the archive created for a particular version and git hash to -/// the releases directory on cloud storage, and updates the metadata for -/// releases. -/// -/// See https://github.com/flutter/flutter/wiki/Release-process for more -/// information on the release process. -class ArchivePublisher { - ArchivePublisher( - this.revision, - this.version, - this.channel, { - this.processManager = const LocalProcessManager(), - this.tempDir, - }) : assert(revision.length == 40, 'Git hash must be 40 characters long (i.e. the entire hash).'); - - /// A git hash describing the revision to publish. It should be the complete - /// hash, not just a prefix. - final String revision; - - /// A version number for the release (e.g. "1.2.3"). - final String version; - - /// The channel to publish to. - // TODO(gspencer): support Channel.beta: it is currently unimplemented. - final Channel channel; - - /// Get the name of the channel as a string. - String get channelName { - switch (channel) { - case Channel.beta: - return 'beta'; - case Channel.dev: - default: - return 'dev'; - } - } - - /// The process manager to use for invoking commands. Typically only - /// used for testing purposes. - final ProcessManager processManager; - - /// The temporary directory used for this publisher. If not set, one will - /// be created, used, and then removed automatically. If set, it will not be - /// deleted when done: that is left to the caller. Typically used by tests. - Directory tempDir; - - static String gsBase = 'gs://flutter_infra'; - static String releaseFolder = '/releases'; - static String baseUrl = 'https://storage.googleapis.com/flutter_infra'; - static String archivePrefix = 'flutter_'; - static String releaseNotesPrefix = 'release_notes_'; - - final String metadataGsPath = '$gsBase$releaseFolder/releases.json'; - - /// Publishes the archive for the given constructor parameters. - bool publishArchive() { - assert(channel == Channel.dev, 'Channel must be dev (beta not yet supported)'); - final List platforms = ['linux', 'mac', 'win']; - final Map metadata = {}; - for (String platform in platforms) { - final String src = _builtArchivePath(platform); - final String dest = _destinationArchivePath(platform); - final String srcGsPath = '$gsBase$src'; - final String destGsPath = '$gsBase$releaseFolder$dest'; - _cloudCopy(srcGsPath, destGsPath); - metadata['${platform}_archive'] = '$channelName/$platform$dest'; - } - metadata['release_date'] = new DateTime.now().toUtc().toIso8601String(); - metadata['version'] = version; - _updateMetadata(metadata); - return true; - } - - /// Checks to make sure the user has access to the Google Storage bucket - /// required to publish. Will throw an [ArchivePublisherException] if not. - void checkForGSUtilAccess() { - // Fetching ACLs requires FULL_CONTROL access. - final ProcessResult result = _runGsUtil(['acl', 'get', metadataGsPath]); - if (result.exitCode != 0) { - throw new ArchivePublisherException( - 'GSUtil cannot get ACLs for metadata file $metadataGsPath', - result, - ); - } - } - - void _updateMetadata(Map metadata) { - final ProcessResult result = _runGsUtil(['cat', metadataGsPath]); - if (result.exitCode != 0) { - throw new ArchivePublisherException( - 'Unable to get existing metadata at $metadataGsPath', result); - } - final String currentMetadata = result.stdout; - if (currentMetadata.isEmpty) { - throw new ArchivePublisherException('Empty metadata received from server', result); - } - Map jsonData; - try { - jsonData = json.decode(currentMetadata); - } on FormatException catch (e) { - throw new ArchivePublisherException('Unable to parse JSON metadata received from cloud: $e'); - } - jsonData['current_$channelName'] = revision; - if (!jsonData.containsKey('releases')) { - jsonData['releases'] = {}; - } - if (jsonData['releases'].containsKey(revision)) { - throw new ArchivePublisherException( - 'Revision $revision already exists in metadata! Aborting.'); - } - jsonData['releases'][revision] = metadata; - final Directory localTempDir = tempDir ?? Directory.systemTemp.createTempSync('flutter_'); - final File tempFile = new File(path.join(localTempDir.absolute.path, 'releases.json')); - const JsonEncoder encoder = const JsonEncoder.withIndent(' '); - tempFile.writeAsStringSync(encoder.convert(jsonData)); - _cloudCopy(tempFile.absolute.path, metadataGsPath); - if (tempDir == null) { - localTempDir.delete(recursive: true); - } - } - - String _getArchiveSuffix(String platform) { - switch (platform) { - case 'linux': - case 'mac': - return '.tar.xz'; - case 'win': - return '.zip'; - default: - assert(false, 'platform $platform not recognized.'); - return null; - } - } - - String _builtArchivePath(String platform) { - final String shortRevision = revision.substring(0, revision.length > 10 ? 10 : revision.length); - final String archivePathBase = '/flutter/$revision/$archivePrefix'; - final String suffix = _getArchiveSuffix(platform); - return '$archivePathBase${platform}_$shortRevision$suffix'; - } - - String _destinationArchivePath(String platform) { - final String archivePathBase = '/$channelName/$platform/$archivePrefix'; - final String suffix = _getArchiveSuffix(platform); - return '$archivePathBase${platform}_$version-$channelName$suffix'; - } - - ProcessResult _runGsUtil(List args) { - return processManager.runSync(['gsutil']..addAll(args)); - } - - void _cloudCopy(String src, String dest) { - final ProcessResult result = _runGsUtil(['cp', src, dest]); - if (result.exitCode != 0) { - throw new ArchivePublisherException('GSUtil copy command failed: ${result.stderr}', result); - } - } -} diff --git a/dev/tools/lib/roll_dev.dart b/dev/tools/lib/roll_dev.dart index bba0de3589..eaa83775cd 100644 --- a/dev/tools/lib/roll_dev.dart +++ b/dev/tools/lib/roll_dev.dart @@ -10,7 +10,6 @@ import 'dart:io'; import 'package:args/args.dart'; -import 'archive_publisher.dart'; const String kIncrement = 'increment'; const String kX = 'x'; @@ -114,19 +113,6 @@ void main(List args) { final String hash = getGitOutput('rev-parse HEAD', 'Get git hash for $commit'); - final ArchivePublisher publisher = new ArchivePublisher(hash, version, Channel.dev); - - // Check for access early so that we don't try to publish things if the - // user doesn't have access to the metadata file. - try { - publisher.checkForGSUtilAccess(); - } on ArchivePublisherException { - print('You do not appear to have the credentials required to update the archive links.'); - print('Make sure you have "gsutil" installed, then run "gsutil config".'); - print('Talk to @gspencergoog for details on which project to use.'); - exit(1); - } - runGit('tag v$version', 'tag the commit with the version label'); // PROMPT @@ -140,17 +126,6 @@ void main(List args) { exit(0); } - // Publish the archive before pushing the tag so that if something fails in - // the publish step, we can clean up. - try { - publisher.publishArchive(); - } on ArchivePublisherException catch (e) { - print('Archive publishing failed.\n$e'); - runGit('tag -d v$version', 'remove the tag that was not published'); - print('The dev roll has been aborted.'); - exit(1); - } - runGit('push upstream v$version', 'publish the version'); runGit('push upstream HEAD:dev', 'land the new version on the "dev" branch'); print('Flutter version $version has been rolled to the "dev" channel!'); diff --git a/dev/tools/test/archive_publisher_test.dart b/dev/tools/test/archive_publisher_test.dart deleted file mode 100644 index da971dc6bb..0000000000 --- a/dev/tools/test/archive_publisher_test.dart +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:io'; - -import 'package:test/test.dart'; - -import 'package:path/path.dart' as path; - -import '../lib/archive_publisher.dart'; -import 'fake_process_manager.dart'; - -void main() { - group('ArchivePublisher', () { - final List emptyStdout = ['']; - FakeProcessManager processManager; - Directory tempDir; - - setUp(() async { - processManager = new FakeProcessManager(); - tempDir = await Directory.systemTemp.createTemp('flutter_'); - }); - - tearDown(() async { - // On Windows, the directory is locked and not able to be deleted, because it is a - // temporary directory. So we just leave some (very small, because we're not actually - // building archives here) trash around to be deleted at the next reboot. - if (!Platform.isWindows) { - await tempDir.delete(recursive: true); - } - }); - - test('calls the right processes', () { - final Map> calls = >{ - 'gsutil acl get gs://flutter_infra/releases/releases.json': emptyStdout, - 'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_linux_deadbeef.tar.xz ' - 'gs://flutter_infra/releases/dev/linux/flutter_linux_1.2.3-dev.tar.xz': emptyStdout, - 'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_mac_deadbeef.tar.xz ' - 'gs://flutter_infra/releases/dev/mac/flutter_mac_1.2.3-dev.tar.xz': emptyStdout, - 'gsutil cp gs://flutter_infra/flutter/deadbeef/flutter_win_deadbeef.zip ' - 'gs://flutter_infra/releases/dev/win/flutter_win_1.2.3-dev.zip': emptyStdout, - 'gsutil cat gs://flutter_infra/releases/releases.json': [ - '''{ - "base_url": "https://storage.googleapis.com/flutter_infra/releases", - "current_beta": "6da8ec6bd0c4801b80d666869e4069698561c043", - "current_dev": "f88c60b38c3a5ef92115d24e3da4175b4890daba", - "releases": { - "6da8ec6bd0c4801b80d666869e4069698561c043": { - "linux_archive": "beta/linux/flutter_linux_0.21.0-beta.tar.xz", - "mac_archive": "beta/mac/flutter_mac_0.21.0-beta.tar.xz", - "windows_archive": "beta/win/flutter_win_0.21.0-beta.tar.xz", - "release_date": "2017-12-19T10:30:00,847287019-08:00", - "release_notes": "beta/release_notes_0.21.0-beta.html", - "version": "0.21.0-beta" - }, - "f88c60b38c3a5ef92115d24e3da4175b4890daba": { - "linux_archive": "dev/linux/flutter_linux_0.22.0-dev.tar.xz", - "mac_archive": "dev/mac/flutter_mac_0.22.0-dev.tar.xz", - "windows_archive": "dev/win/flutter_win_0.22.0-dev.tar.xz", - "release_date": "2018-01-19T13:30:09,728487019-08:00", - "release_notes": "dev/release_notes_0.22.0-dev.html", - "version": "0.22.0-dev" - } - } -} -'''], - 'gsutil cp ${tempDir.path}/releases.json gs://flutter_infra/releases/releases.json': - emptyStdout, - }; - processManager.setResults(calls); - new ArchivePublisher('deadbeef', '1.2.3', Channel.dev, - processManager: processManager, tempDir: tempDir) - ..publishArchive(); - processManager.verifyCalls(calls.keys); - final File outputFile = new File(path.join(tempDir.path, 'releases.json')); - expect(outputFile.existsSync(), isTrue); - final String contents = outputFile.readAsStringSync(); - expect(contents, contains('"current_dev": "deadbeef"')); - expect(contents, contains('"deadbeef": {')); - }); - }); -}