zan1456/flutter
1
0
Fork 0
forked from firka/flutter
Code Pull Requests Activity

Re-land: "Remove Cocoon from dev/devicelab, keeping Skia perf stats upload" (#165755)

Browse Source 
🚫 **Merge after**
https://flutter-review.googlesource.com/c/recipes/+/64300.

---

This reverts commit 0a2d9f5658.
This commit is contained in:
Matan Lurey
2025-03-22 14:06:17 -07:00
committed by GitHub
parent 0a2d9f5658
commit 5840236ae5
8 changed files with 247 additions and 770 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
dev/devicelab/lib/command/upload_metrics.dart Normal file
View File

@@ -0,0 +1,42 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:args/command_runner.dart';
import '../framework/metrics_center.dart';
class UploadMetricsCommand extends Command<void> {
UploadMetricsCommand() {
argParser.addOption('results-file', help: 'Test results JSON to upload to Cocoon.');
argParser.addOption('commit-time', help: 'Commit time in UNIX timestamp');
argParser.addOption(
'task-name',
help: '[Flutter infrastructure] Name of the task being run on.',
);
argParser.addOption(
'benchmark-tags',
help: '[Flutter infrastructure] Benchmark tags to surface on Skia Perf',
);
}
@override
String get name => 'upload-metrics';
@override
String get description => '[Flutter infrastructure] Upload results data to Cocoon/Skia Perf';
@override
Future<void> run() async {
final String? resultsPath = argResults!['results-file'] as String?;
final String? commitTime = argResults!['commit-time'] as String?;
final String? taskName = argResults!['task-name'] as String?;
final String? benchmarkTags = argResults!['benchmark-tags'] as String?;
// Upload metrics to skia perf from test runner when `resultsPath` is specified.
if (resultsPath != null) {
await uploadToSkiaPerf(resultsPath, commitTime, taskName, benchmarkTags);
print('Successfully uploaded metrics to skia perf');
}
}
}

84
dev/devicelab/lib/command/upload_results.dart
View File

@@ -1,84 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:args/command_runner.dart';
import '../framework/metrics_center.dart';
class UploadResultsCommand extends Command<void> {
UploadResultsCommand() {
argParser.addOption('results-file', help: 'Test results JSON to upload to Cocoon.');
argParser.addOption(
'service-account-token-file',
help: 'Authentication token for uploading results.',
);
argParser.addOption(
'test-flaky',
help: 'Flag to show whether the test is flaky: "True" or "False"',
);
argParser.addOption(
'git-branch',
help:
'[Flutter infrastructure] Git branch of the current commit. LUCI\n'
'checkouts run in detached HEAD state, so the branch must be passed.',
);
argParser.addOption(
'luci-builder',
help: '[Flutter infrastructure] Name of the LUCI builder being run on.',
);
argParser.addOption(
'task-name',
help: '[Flutter infrastructure] Name of the task being run on.',
);
argParser.addOption(
'benchmark-tags',
help: '[Flutter infrastructure] Benchmark tags to surface on Skia Perf',
);
argParser.addOption('test-status', help: 'Test status: Succeeded|Failed');
argParser.addOption('commit-time', help: 'Commit time in UNIX timestamp');
argParser.addOption(
'builder-bucket',
help: '[Flutter infrastructure] Luci builder bucket the test is running in.',
);
}
@override
String get name => 'upload-metrics';
@override
String get description => '[Flutter infrastructure] Upload results data to Cocoon/Skia Perf';
@override
Future<void> run() async {
final String? resultsPath = argResults!['results-file'] as String?;
// final String? serviceAccountTokenFile = argResults!['service-account-token-file'] as String?;
// final String? testFlakyStatus = argResults!['test-flaky'] as String?;
final String? gitBranch = argResults!['git-branch'] as String?;
final String? builderName = argResults!['luci-builder'] as String?;
final String? testStatus = argResults!['test-status'] as String?;
final String? commitTime = argResults!['commit-time'] as String?;
final String? taskName = argResults!['task-name'] as String?;
final String? benchmarkTags = argResults!['benchmark-tags'] as String?;
final String? builderBucket = argResults!['builder-bucket'] as String?;
// Upload metrics to skia perf from test runner when `resultsPath` is specified.
if (resultsPath != null) {
await uploadToSkiaPerf(resultsPath, commitTime, taskName, benchmarkTags);
print('Successfully uploaded metrics to skia perf');
}
print(
'Intentionally skipping /api/update-task-status ($gitBranch/$builderName/$testStatus/$builderBucket) because yjbanov@ said so',
);
// final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
// return cocoon.sendTaskStatus(
// resultsPath: resultsPath,
// isTestFlaky: testFlakyStatus == 'True',
// gitBranch: gitBranch,
// builderName: builderName,
// testStatus: testStatus,
// builderBucket: builderBucket,
// );
}
}

265
dev/devicelab/lib/framework/cocoon.dart
View File

@@ -1,265 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show Encoding, json;
import 'dart:io';
import 'package:file/file.dart';
import 'package:file/local.dart';
import 'package:http/http.dart';
import 'package:logging/logging.dart';
import 'package:meta/meta.dart';
import 'task_result.dart';
import 'utils.dart';
typedef ProcessRunSync =
ProcessResult Function(
String,
List<String>, {
Map<String, String>? environment,
bool includeParentEnvironment,
bool runInShell,
Encoding? stderrEncoding,
Encoding? stdoutEncoding,
String? workingDirectory,
});
/// Class for test runner to interact with Flutter's infrastructure service, Cocoon.
///
/// Cocoon assigns bots to run these devicelab tasks on real devices.
/// To retrieve these results, the test runner needs to send results back so the database can be updated.
class Cocoon {
Cocoon({
String? serviceAccountTokenPath,
@visibleForTesting Client? httpClient,
@visibleForTesting this.fs = const LocalFileSystem(),
@visibleForTesting this.processRunSync = Process.runSync,
@visibleForTesting this.requestRetryLimit = 5,
@visibleForTesting this.requestTimeoutLimit = 30,
}) : _httpClient = AuthenticatedCocoonClient(
serviceAccountTokenPath,
httpClient: httpClient,
filesystem: fs,
);
/// Client to make http requests to Cocoon.
final AuthenticatedCocoonClient _httpClient;
final ProcessRunSync processRunSync;
/// Url used to send results to.
static const String baseCocoonApiUrl = 'https://flutter-dashboard.appspot.com/api';
/// Threshold to auto retry a failed test.
static const int retryNumber = 2;
/// Underlying [FileSystem] to use.
final FileSystem fs;
static final Logger logger = Logger('CocoonClient');
@visibleForTesting
final int requestRetryLimit;
@visibleForTesting
final int requestTimeoutLimit;
String get commitSha => _commitSha ?? _readCommitSha();
String? _commitSha;
/// Parse the local repo for the current running commit.
String _readCommitSha() {
final ProcessResult result = processRunSync('git', <String>['rev-parse', 'HEAD']);
if (result.exitCode != 0) {
throw CocoonException(result.stderr as String);
}
return _commitSha = result.stdout as String;
}
/// Update test status to Cocoon.
///
/// Flutter infrastructure's workflow is:
/// 1. Run DeviceLab test
/// 2. Request service account token from luci auth (valid for at least 3 minutes)
/// 3. Update test status from (1) to Cocoon
///
/// The `resultsPath` is not available for all tests. When it doesn't show up, we
/// need to append `CommitBranch`, `CommitSha`, and `BuilderName`.
Future<void> sendTaskStatus({
String? resultsPath,
bool? isTestFlaky,
String? gitBranch,
String? builderName,
String? testStatus,
String? builderBucket,
}) async {
Map<String, dynamic> resultsJson = <String, dynamic>{};
if (resultsPath != null) {
final File resultFile = fs.file(resultsPath);
resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
} else {
resultsJson['CommitBranch'] = gitBranch;
resultsJson['CommitSha'] = commitSha;
resultsJson['BuilderName'] = builderName;
resultsJson['NewStatus'] = testStatus;
}
resultsJson['TestFlaky'] = isTestFlaky ?? false;
if (_shouldUpdateCocoon(resultsJson, builderBucket ?? 'prod')) {
await retry(
() async =>
_sendUpdateTaskRequest(resultsJson).timeout(Duration(seconds: requestTimeoutLimit)),
retryIf:
(Exception e) => e is SocketException || e is TimeoutException || e is ClientException,
maxAttempts: requestRetryLimit,
);
}
}
/// Only post-submit tests on `master` are allowed to update in cocoon.
bool _shouldUpdateCocoon(Map<String, dynamic> resultJson, String builderBucket) {
const List<String> supportedBranches = <String>['master'];
return supportedBranches.contains(resultJson['CommitBranch']) && builderBucket == 'prod';
}
/// Write the given parameters into an update task request and store the JSON in [resultsPath].
Future<void> writeTaskResultToFile({
String? builderName,
String? gitBranch,
required TaskResult result,
required String resultsPath,
}) async {
final Map<String, dynamic> updateRequest = _constructUpdateRequest(
gitBranch: gitBranch,
builderName: builderName,
result: result,
);
final File resultFile = fs.file(resultsPath);
if (resultFile.existsSync()) {
resultFile.deleteSync();
}
logger.fine('Writing results: ${json.encode(updateRequest)}');
resultFile.createSync();
resultFile.writeAsStringSync(json.encode(updateRequest));
}
Map<String, dynamic> _constructUpdateRequest({
String? builderName,
required TaskResult result,
String? gitBranch,
}) {
final Map<String, dynamic> updateRequest = <String, dynamic>{
'CommitBranch': gitBranch,
'CommitSha': commitSha,
'BuilderName': builderName,
'NewStatus': result.succeeded ? 'Succeeded' : 'Failed',
};
logger.fine('Update request: $updateRequest');
// Make a copy of result data because we may alter it for validation below.
updateRequest['ResultData'] = result.data;
final List<String> validScoreKeys = <String>[];
if (result.benchmarkScoreKeys != null) {
for (final String scoreKey in result.benchmarkScoreKeys!) {
final Object score = result.data![scoreKey] as Object;
if (score is num) {
// Convert all metrics to double, which provide plenty of precision
// without having to add support for multiple numeric types in Cocoon.
result.data![scoreKey] = score.toDouble();
validScoreKeys.add(scoreKey);
}
}
}
updateRequest['BenchmarkScoreKeys'] = validScoreKeys;
return updateRequest;
}
Future<void> _sendUpdateTaskRequest(Map<String, dynamic> postBody) async {
logger.info('Attempting to send update task request to Cocoon.');
final Map<String, dynamic> response = await _sendCocoonRequest('update-task-status', postBody);
if (response['Name'] != null) {
logger.info('Updated Cocoon with results from this task');
} else {
logger.info(response);
logger.severe('Failed to updated Cocoon with results from this task');
}
}
/// Make an API request to Cocoon.
Future<Map<String, dynamic>> _sendCocoonRequest(String apiPath, [dynamic jsonData]) async {
final Uri url = Uri.parse('$baseCocoonApiUrl/$apiPath');
/// Retry requests to Cocoon as sometimes there are issues with the servers, such
/// as version changes to the backend, datastore issues, or latency issues.
final Response response = await retry(
() => _httpClient.post(url, body: json.encode(jsonData)),
retryIf:
(Exception e) => e is SocketException || e is TimeoutException || e is ClientException,
maxAttempts: requestRetryLimit,
);
return json.decode(response.body) as Map<String, dynamic>;
}
}
/// [HttpClient] for sending authenticated requests to Cocoon.
class AuthenticatedCocoonClient extends BaseClient {
AuthenticatedCocoonClient(
this._serviceAccountTokenPath, {
@visibleForTesting Client? httpClient,
@visibleForTesting FileSystem? filesystem,
}) : _delegate = httpClient ?? Client(),
_fs = filesystem ?? const LocalFileSystem();
/// Authentication token to have the ability to upload and record test results.
///
/// This is intended to only be passed on automated runs on LUCI post-submit.
final String? _serviceAccountTokenPath;
/// Underlying [HttpClient] to send requests to.
final Client _delegate;
/// Underlying [FileSystem] to use.
final FileSystem _fs;
/// Value contained in the service account token file that can be used in http requests.
String get serviceAccountToken => _serviceAccountToken ?? _readServiceAccountTokenFile();
String? _serviceAccountToken;
/// Get [serviceAccountToken] from the given service account file.
String _readServiceAccountTokenFile() {
return _serviceAccountToken = _fs.file(_serviceAccountTokenPath).readAsStringSync().trim();
}
@override
Future<StreamedResponse> send(BaseRequest request) async {
request.headers['Service-Account-Token'] = serviceAccountToken;
final StreamedResponse response = await _delegate.send(request);
if (response.statusCode != 200) {
throw ClientException(
'AuthenticatedClientError:\n'
' URI: ${request.url}\n'
' HTTP Status: ${response.statusCode}\n'
' Response body:\n'
'${(await Response.fromStream(response)).body}',
request.url,
);
}
return response;
}
}
class CocoonException implements Exception {
CocoonException(this.message);
/// The message to show to the issuer to explain the error.
final String message;
@override
String toString() => 'CocoonException: $message';
}

119
dev/devicelab/lib/framework/metrics_result_writer.dart Normal file
View File

@@ -0,0 +1,119 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show Encoding, json;
import 'dart:io';
import 'package:file/file.dart';
import 'package:file/local.dart';
import 'package:logging/logging.dart';
import 'package:meta/meta.dart';
import 'task_result.dart';
/// Class for test runner to write device-lab metrics results for Skia Perf.
interface class MetricsResultWriter {
MetricsResultWriter({
@visibleForTesting this.fs = const LocalFileSystem(),
@visibleForTesting this.processRunSync = Process.runSync,
});
final ProcessResult Function(
String,
List<String>, {
Map<String, String>? environment,
bool includeParentEnvironment,
bool runInShell,
Encoding? stderrEncoding,
Encoding? stdoutEncoding,
String? workingDirectory,
})
processRunSync;
/// Threshold to auto retry a failed test.
static const int retryNumber = 2;
/// Underlying [FileSystem] to use.
final FileSystem fs;
static final Logger logger = Logger('CocoonClient');
String get commitSha => _commitSha ?? _readCommitSha();
String? _commitSha;
/// Parse the local repo for the current running commit.
String _readCommitSha() {
final ProcessResult result = processRunSync('git', <String>['rev-parse', 'HEAD']);
if (result.exitCode != 0) {
throw CocoonException(result.stderr as String);
}
return _commitSha = result.stdout as String;
}
/// Write the given parameters into an update task request and store the JSON in [resultsPath].
Future<void> writeTaskResultToFile({
String? builderName,
String? gitBranch,
required TaskResult result,
required String resultsPath,
}) async {
final Map<String, dynamic> updateRequest = _constructUpdateRequest(
gitBranch: gitBranch,
builderName: builderName,
result: result,
);
final File resultFile = fs.file(resultsPath);
if (resultFile.existsSync()) {
resultFile.deleteSync();
}
logger.fine('Writing results: ${json.encode(updateRequest)}');
resultFile.createSync();
resultFile.writeAsStringSync(json.encode(updateRequest));
}
Map<String, dynamic> _constructUpdateRequest({
String? builderName,
required TaskResult result,
String? gitBranch,
}) {
final Map<String, dynamic> updateRequest = <String, dynamic>{
'CommitBranch': gitBranch,
'CommitSha': commitSha,
'BuilderName': builderName,
'NewStatus': result.succeeded ? 'Succeeded' : 'Failed',
};
logger.fine('Update request: $updateRequest');
// Make a copy of result data because we may alter it for validation below.
updateRequest['ResultData'] = result.data;
final List<String> validScoreKeys = <String>[];
if (result.benchmarkScoreKeys != null) {
for (final String scoreKey in result.benchmarkScoreKeys!) {
final Object score = result.data![scoreKey] as Object;
if (score is num) {
// Convert all metrics to double, which provide plenty of precision
// without having to add support for multiple numeric types in Cocoon.
result.data![scoreKey] = score.toDouble();
validScoreKeys.add(scoreKey);
}
}
}
updateRequest['BenchmarkScoreKeys'] = validScoreKeys;
return updateRequest;
}
}
class CocoonException implements Exception {
CocoonException(this.message);
/// The message to show to the issuer to explain the error.
final String message;
@override
String toString() => 'CocoonException: $message';
}

6
dev/devicelab/lib/framework/runner.dart
View File

@@ -10,8 +10,8 @@ import 'package:meta/meta.dart';
import 'package:vm_service/vm_service.dart';
import 'package:vm_service/vm_service_io.dart';
import 'cocoon.dart';
import 'devices.dart';
import 'metrics_result_writer.dart';
import 'task_result.dart';
import 'utils.dart';
@@ -49,7 +49,7 @@ Future<void> runTasks(
for (final String taskName in taskNames) {
TaskResult result = TaskResult.success(null);
int failureCount = 0;
while (failureCount <= Cocoon.retryNumber) {
while (failureCount <= MetricsResultWriter.retryNumber) {
result = await rerunTask(
taskName,
deviceId: deviceId,
@@ -137,7 +137,7 @@ Future<TaskResult> rerunTask(
section('Finished task "$taskName"');
if (resultsPath != null) {
final Cocoon cocoon = Cocoon();
final MetricsResultWriter cocoon = MetricsResultWriter();
await cocoon.writeTaskResultToFile(
builderName: luciBuilder,
gitBranch: gitBranch,