zan1456/flutter
1
0
Fork 0
forked from firka/flutter
Code Pull Requests Activity

Better filtering for Android scenario_app runner. (flutter/engine#50937)

Browse Source 
_🍴 'd from https://github.com/flutter/engine/pull/50933, will rebase
when merged._

Closes https://github.com/flutter/flutter/issues/143458.

A picture is a 1000 words:

![Screenshot 2024-02-23 at 7 01
29 PM](https://github.com/flutter/engine/assets/168174/7254b3be-cc49-4bad-ae43-e61ac4a853ad)

This is still noisy, but at least all the output appears to be part of
the execution.

As you recall, the full logs are always available in the
FLUTTER_LOGS_DIR output.
This commit is contained in:
Matan Lurey
2024-02-26 11:36:10 -08:00
committed by GitHub
parent d1d7c08ce9
commit dec92d85f2
4 changed files with 262 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
engine/src/flutter/testing/scenario_app/bin/run_android_tests.dart
View File

@@ -47,48 +47,52 @@ void main(List<String> args) async {
..addOption(
'adb',
help: 'Path to the adb tool',
defaultsTo: engine != null ? join(
engine.srcDir.path,
'third_party',
'android_tools',
'sdk',
'platform-tools',
'adb',
) : null,
defaultsTo: engine != null
? join(
engine.srcDir.path,
'third_party',
'android_tools',
'sdk',
'platform-tools',
'adb',
)
: null,
)
..addOption(
'ndk-stack',
help: 'Path to the ndk-stack tool',
defaultsTo: engine != null ? join(
engine.srcDir.path,
'third_party',
'android_tools',
'ndk',
'prebuilt',
() {
if (Platform.isLinux) {
return 'linux-x86_64';
} else if (Platform.isMacOS) {
return 'darwin-x86_64';
} else if (Platform.isWindows) {
return 'windows-x86_64';
} else {
throw UnsupportedError('Unsupported platform: ${Platform.operatingSystem}');
}
}(),
'bin',
'ndk-stack',
) : null,
defaultsTo: engine != null
? join(
engine.srcDir.path,
'third_party',
'android_tools',
'ndk',
'prebuilt',
() {
if (Platform.isLinux) {
return 'linux-x86_64';
} else if (Platform.isMacOS) {
return 'darwin-x86_64';
} else if (Platform.isWindows) {
return 'windows-x86_64';
} else {
throw UnsupportedError('Unsupported platform: ${Platform.operatingSystem}');
}
}(),
'bin',
'ndk-stack',
)
: null,
)
..addOption(
'out-dir',
help: 'Out directory',
defaultsTo:
engine?.
outputs().
where((Output o) => basename(o.path.path).startsWith('android_')).
firstOrNull?.
path.path,
defaultsTo: engine
?.outputs()
.where((Output o) => basename(o.path.path).startsWith('android_'))
.firstOrNull
?.path
.path,
)
..addOption(
'smoke-test',
@@ -106,14 +110,16 @@ void main(List<String> args) async {
..addOption(
'output-contents-golden',
help: 'Path to a file that contains the expected filenames of golden files.',
defaultsTo: engine != null ? join(
engine.srcDir.path,
'flutter',
'testing',
'scenario_app',
'android',
'expected_golden_output.txt',
) : null,
defaultsTo: engine != null
? join(
engine.srcDir.path,
'flutter',
'testing',
'scenario_app',
'android',
'expected_golden_output.txt',
)
: null,
)
..addOption(
'impeller-backend',
@@ -124,8 +130,8 @@ void main(List<String> args) async {
..addOption(
'logs-dir',
help: 'The directory to store the logs and screenshots. Defaults to '
'the value of the FLUTTER_LOGS_DIR environment variable, if set, '
'otherwise it defaults to a path within out-dir.',
'the value of the FLUTTER_LOGS_DIR environment variable, if set, '
'otherwise it defaults to a path within out-dir.',
defaultsTo: Platform.environment['FLUTTER_LOGS_DIR'],
);
@@ -153,7 +159,10 @@ void main(List<String> args) async {
final String? contentsGolden = results['output-contents-golden'] as String?;
final _ImpellerBackend? impellerBackend = _ImpellerBackend.tryParse(results['impeller-backend'] as String?);
if (enableImpeller && impellerBackend == null) {
panic(<String>['invalid graphics-backend', results['impeller-backend'] as String? ?? '<null>']);
panic(<String>[
'invalid graphics-backend',
results['impeller-backend'] as String? ?? '<null>'
]);
}
final Directory logsDir = Directory(results['logs-dir'] as String? ?? join(outDir.path, 'scenario_app', 'logs'));
final String? ndkStack = results['ndk-stack'] as String?;
@@ -215,7 +224,10 @@ Future<void> _run({
const ProcessManager pm = LocalProcessManager();
if (!outDir.existsSync()) {
panic(<String>['out-dir does not exist: $outDir', 'make sure to build the selected engine variant']);
panic(<String>[
'out-dir does not exist: $outDir',
'make sure to build the selected engine variant'
]);
}
if (!adb.existsSync()) {
@@ -236,11 +248,17 @@ Future<void> _run({
log('writing logs and screenshots to ${logsDir.path}');
if (!testApk.existsSync()) {
panic(<String>['test apk does not exist: ${testApk.path}', 'make sure to build the selected engine variant']);
panic(<String>[
'test apk does not exist: ${testApk.path}',
'make sure to build the selected engine variant'
]);
}
if (!appApk.existsSync()) {
panic(<String>['app apk does not exist: ${appApk.path}', 'make sure to build the selected engine variant']);
panic(<String>[
'app apk does not exist: ${appApk.path}',
'make sure to build the selected engine variant'
]);
}
// Start a TCP socket in the host, and forward it to the device that runs the tests.
@@ -248,7 +266,7 @@ Future<void> _run({
// for the screenshots.
// On LUCI, the host uploads the screenshots to Skia Gold.
SkiaGoldClient? skiaGoldClient;
late ServerSocket server;
late ServerSocket server;
final List<Future<void>> pendingComparisons = <Future<void>>[];
await step('Starting server...', () async {
server = await ServerSocket.bind(InternetAddress.anyIPv4, _tcpPort);
@@ -259,7 +277,8 @@ Future<void> _run({
if (verbose) {
stdout.writeln('client connected ${client.remoteAddress.address}:${client.remotePort}');
}
client.transform(const ScreenshotBlobTransformer()).listen((Screenshot screenshot) {
client.transform(const ScreenshotBlobTransformer()).listen(
(Screenshot screenshot) {
final String fileName = screenshot.filename;
final Uint8List fileContent = screenshot.fileContent;
if (verbose) {
@@ -277,18 +296,15 @@ Future<void> _run({
}
if (isSkiaGoldClientAvailable) {
final Future<void> comparison = skiaGoldClient!
.addImg(fileName, goldenFile,
screenshotSize: screenshot.pixelCount)
.catchError((dynamic err) {
panic(<String>['skia gold comparison failed: $err']);
});
.addImg(fileName, goldenFile, screenshotSize: screenshot.pixelCount)
.catchError((dynamic err) {
panic(<String>['skia gold comparison failed: $err']);
});
pendingComparisons.add(comparison);
}
},
onError: (dynamic err) {
}, onError: (dynamic err) {
panic(<String>['error while receiving bytes: $err']);
},
cancelOnError: true);
}, cancelOnError: true);
});
});
@@ -311,27 +327,38 @@ Future<void> _run({
final (Future<int> logcatExitCode, Stream<String> logcatOutput) = getProcessStreams(logcatProcess);
logcatProcessExitCode = logcatExitCode;
String? filterProcessId;
logcatOutput.listen((String line) {
// Always write to the full log.
logcat.writeln(line);
// Conditionally parse and write to stderr.
final AdbLogLine? adbLogLine = AdbLogLine.tryParse(line);
switch (adbLogLine?.process) {
case null:
break;
case 'ActivityManager':
// These are mostly noise, i.e. "D ActivityManager: freezing 24632 com.blah".
if (adbLogLine!.severity == 'D') {
break;
}
// TODO(matanlurey): Figure out why this isn't 'flutter.scenario' or similar.
// Also, why is there two different names?
case 'utter.scenario':
case 'utter.scenarios':
case 'flutter':
case 'FlutterJNI':
log('[adb] $line');
if (verbose || adbLogLine == null) {
log(line);
return;
}
// If we haven't already found a process ID, try to find one.
// The process ID will help us filter out logs from other processes.
filterProcessId ??= adbLogLine.tryParseProcess();
// If this is a "verbose" log, possibly skip it.
final bool isVerbose = adbLogLine.isVerbose(filterProcessId: filterProcessId);
if (isVerbose || filterProcessId == null) {
// We've requested verbose output, so print everything.
if (verbose) {
adbLogLine.printFormatted();
}
return;
}
// It's a non-verbose log, so print it.
adbLogLine.printFormatted();
}, onError: (Object? err) {
if (verbose) {
logWarning('logcat stream error: $err');
}
});
});
@@ -364,10 +391,7 @@ Future<void> _run({
log('using dimensions: ${json.encode(dimensions)}');
skiaGoldClient = SkiaGoldClient(
outDir,
dimensions: <String, String>{
'AndroidAPILevel': connectedDeviceAPILevel,
'GraphicsBackend': enableImpeller ? 'impeller-${impellerBackend!.name}' : 'skia',
},
dimensions: dimensions,
);
});
@@ -412,11 +436,9 @@ Future<void> _run({
'am',
'instrument',
'-w',
if (smokeTestFullPath != null)
'-e class $smokeTestFullPath',
if (smokeTestFullPath != null) '-e class $smokeTestFullPath',
'dev.flutter.scenarios.test/dev.flutter.TestRunner',
if (enableImpeller)
'-e enable-impeller',
if (enableImpeller) '-e enable-impeller',
if (impellerBackend != null)
'-e impeller-backend ${impellerBackend.name}',
]);
@@ -465,7 +487,8 @@ Future<void> _run({
final int exitCode = await pm.runAndForward(<String>[
adb.path,
'reverse',
'--remove', 'tcp:3000',
'--remove',
'tcp:3000',
]);
if (exitCode != 0) {
panic(<String>['could not unforward port']);
@@ -473,14 +496,16 @@ Future<void> _run({
});
await step('Uninstalling app APK...', () async {
final int exitCode = await pm.runAndForward(<String>[adb.path, 'uninstall', 'dev.flutter.scenarios']);
final int exitCode = await pm.runAndForward(
<String>[adb.path, 'uninstall', 'dev.flutter.scenarios']);
if (exitCode != 0) {
panic(<String>['could not uninstall app apk']);
}
});
await step('Uninstalling test APK...', () async {
final int exitCode = await pm.runAndForward(<String>[adb.path, 'uninstall', 'dev.flutter.scenarios.test']);
final int exitCode = await pm.runAndForward(
<String>[adb.path, 'uninstall', 'dev.flutter.scenarios.test']);
if (exitCode != 0) {
panic(<String>['could not uninstall app apk']);
}

105
engine/src/flutter/testing/scenario_app/bin/utils/adb_logcat_filtering.dart
View File

@@ -1,3 +1,7 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
/// Some notes about filtering `adb logcat` output, especially as a result of
/// running `adb shell` to instrument the app and test scripts, as it's
/// non-trivial and error-prone.
@@ -26,6 +30,8 @@
/// See also: <https://developer.android.com/tools/logcat>.
library;
import 'logs.dart';
/// Represents a line of `adb logcat` output parsed into a structured form.
///
/// For example the line:
@@ -40,14 +46,15 @@ library;
/// with lazy parsing.
extension type const AdbLogLine._(Match _match) {
// RegEx that parses into the following groups:
// 1. Everything up to the severity (I, W, E, etc.).
// In other words, any whitespace, numbers, hyphens, colons, and periods.
// 2. The severity (a single uppercase letter).
// 3. The name of the process (up to the colon).
// 4. The message (after the colon).
// 1. The time of the log message, such as `02-22 13:54:39.839`.
// 2. The process ID.
// 3. The thread ID.
// 4. The character representing the severity of the log message, such as `I`.
// 5. The tag, such as `ActivityManager`.
// 6. The actual log message.
//
// This regex is simple versus being more precise. Feel free to improve it.
static final RegExp _pattern = RegExp(r'([^A-Z]*)([A-Z])\s([^:]*)\:\s(.*)');
static final RegExp _pattern = RegExp(r'(\d+-\d+\s[\d|:]+\.\d+)\s+(\d+)\s+(\d+)\s(\w)\s(\S+)\s*:\s*(.*)');
/// Parses the given [adbLogCatLine] into a structured form.
///
@@ -57,15 +64,91 @@ extension type const AdbLogLine._(Match _match) {
return match == null ? null : AdbLogLine._(match);
}
/// Tries to parse the process that was started, if the log line is about it.
String? tryParseProcess() {
if (name == 'ActivityManager' && message.startsWith('Start proc')) {
// Start proc 6840:d
final RegExpMatch? match = RegExp(r'Start proc (\d+):').firstMatch(message);
return match?.group(1);
}
return null;
}
/// Returns `true` if the log line is verbose.
bool isVerbose({String? filterProcessId}) => !_isRelevant(filterProcessId: filterProcessId);
bool _isRelevant({String? filterProcessId}) {
// Fatal errors are always useful.
if (severity == 'F') {
return true;
}
// Debug logs are rarely useful.
if (severity == 'D') {
return false;
}
// These are "known" noise tags.
if (const <String>{
'MonitoringInstr',
'ResourceExtractor',
'THREAD_STATE',
'ziparchive',
}.contains(name)) {
return false;
}
// These are "known" tags useful for debugging.
if (const <String>{
'utter.scenario',
'utter.scenarios',
'TestRunner',
}.contains(name)) {
return true;
}
// If a process ID is specified, exclude logs _not_ from that process.
if (filterProcessId != null && process != filterProcessId) {
return false;
}
// And... whatever, include anything with the word "flutter".
return name.toLowerCase().contains('flutter') || message.toLowerCase().contains('flutter');
}
/// Logs the line to the console.
void printFormatted() {
final String formatted = '$time [$severity] $name: $message';
if (severity == 'W' || severity == 'E' || severity == 'F') {
logWarning(formatted);
} else if (name == 'TestRunner') {
logImportant(formatted);
} else {
log(formatted);
}
}
/// The full line of `adb logcat` output.
String get line => _match.group(0)!;
/// The character representing the severity of the log message, such as `I`.
String get severity => _match.group(2)!;
/// The time of the log message, such as `02-22 13:54:39.839`.
String get time => _match.group(1)!;
/// The process name, such as `ActivityManager`.
String get process => _match.group(3)!;
/// The process ID.
String get process => _match.group(2)!;
/// The thread ID.
String get thread => _match.group(3)!;
/// The character representing the severity of the log message, such as `I`.
String get severity => _match.group(4)!;
/// The tag, such as `ActivityManager`.
String get name => _match.group(5)!;
/// The actual log message.
String get message => _match.group(4)!;
String get message => _match.group(6)!;
String toDebugString() {
return 'AdbLogLine(time: $time, process: $process, thread: $thread, severity: $severity, name: $name, message: $message)';
}
}

17
engine/src/flutter/testing/scenario_app/bin/utils/logs.dart
View File

@@ -7,6 +7,7 @@ import 'dart:io';
bool _supportsAnsi = stdout.supportsAnsiEscapes;
String _green = _supportsAnsi ? '\u001b[1;32m' : '';
String _red = _supportsAnsi ? '\u001b[31m' : '';
String _yellow = _supportsAnsi ? '\u001b[33m' : '';
String _gray = _supportsAnsi ? '\u001b[90m' : '';
String _reset = _supportsAnsi? '\u001B[0m' : '';
@@ -22,15 +23,27 @@ Future<void> step(String msg, Future<void> Function() fn) async {
}
}
void _logWithColor(String color, String msg) {
stdout.writeln('$color$msg$_reset');
}
void log(String msg) {
stdout.writeln('$_gray$msg$_reset');
_logWithColor(_gray, msg);
}
void logImportant(String msg) {
stdout.writeln(msg);
}
void logWarning(String msg) {
_logWithColor(_yellow, msg);
}
final class Panic extends Error {}
Never panic(List<String> messages) {
for (final String message in messages) {
stderr.writeln('$_red$message$_reset');
_logWithColor(_red, message);
}
throw Panic();
}

43
engine/src/flutter/testing/scenario_app/tool/logcat_reader.dart Normal file
View File

@@ -0,0 +1,43 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io' as io;
// It's bad to import a file from `bin` into `tool`.
// However this tool is not very important, so delete it if necessary.
import '../bin/utils/adb_logcat_filtering.dart';
/// A tiny tool to read saved `adb logcat` output and perform some analysis.
///
/// This tool is not meant to be a full-fledged logcat reader. It's just a
/// simple tool that uses the [AdbLogLine] extension type to parse results of
/// `adb logcat` and explain what log tag names are most common.
void main(List<String> args) {
if (args case [final String path]) {
final List<AdbLogLine> parsed = io.File(path)
.readAsLinesSync()
.map(AdbLogLine.tryParse)
.whereType<AdbLogLine>()
// Filter out all debug logs.
.where((AdbLogLine line) => line.severity != 'D')
.toList();
final Map<String, int> tagCounts = <String, int>{};
for (final AdbLogLine line in parsed) {
tagCounts[line.name] = (tagCounts[line.name] ?? 0) + 1;
}
// Print in order of most common to least common.
final List<MapEntry<String, int>> sorted = tagCounts.entries.toList()
..sort((MapEntry<String, int> a, MapEntry<String, int> b) => b.value.compareTo(a.value));
for (final MapEntry<String, int> entry in sorted) {
print("'${entry.key}', // ${entry.value}");
}
return;
}
print('Usage: logcat_reader.dart <path-to-logcat-output>');
io.exitCode = 1;
}