From ec2ab541dd71e8ee800b190671b476b80311edb6 Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Fri, 15 Nov 2024 14:54:06 -0800 Subject: [PATCH] Switch `flutter_build_apk_health_tests` to use a subset of current tests. (#159004) Towards https://github.com/flutter/flutter/issues/159000 as part of debugging issues such as https://github.com/flutter/flutter/issues/158560. On a `bringup: true` shard, run a number of `flutter build apk` tests in succession to try and routinely trigger timeouts and crashes, so we can test other hypotheses on how to fix this problem (i.e. potentially around increasing memory, changing daemon configuration, aggressively killing processes, etc). --- dev/bots/test.dart | 11 ++- dev/bots/utils.dart | 6 ++ .../test/flutter_build_apk.shard/README.md | 3 - .../flutter_build_apk_test.dart | 87 ------------------- .../android_e2e_api_test.dart | 4 + ...d_gradle_deprecated_plugin_apply_test.dart | 4 + ...tter_plugins_strings_in_comments_test.dart | 4 + ...roid_gradle_print_build_variants_test.dart | 4 + ...android_plugin_example_app_build_test.dart | 4 + ...ter_build_with_compilation_error_test.dart | 4 + .../native_assets_agp_version_test.dart | 2 + .../isolated/native_assets_test.dart | 3 + ...e_assets_without_cbuild_assemble_test.dart | 5 ++ 13 files changed, 45 insertions(+), 96 deletions(-) delete mode 100644 packages/flutter_tools/test/flutter_build_apk.shard/README.md delete mode 100644 packages/flutter_tools/test/flutter_build_apk.shard/flutter_build_apk_test.dart diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 7073a9f3b0..0c9ff764d2 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -236,16 +236,15 @@ Future _runIntegrationToolTests() async { } Future _runFlutterBuildApkHealthTests() async { - final List allTests = Directory(path.join(_toolsPath, 'test', 'flutter_build_apk.shard')) - .listSync(recursive: true).whereType() - .map((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath)) - .where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList(); - await runDartTest( _toolsPath, forceSingleCore: true, - testPaths: selectIndexOfTotalSubshard(allTests), + testPaths: [ + path.join(_toolsPath, 'test', 'integration.shard'), + ], collectMetrics: true, + runSkipped: true, + tags: ['flutter-build-apk'], ); } diff --git a/dev/bots/utils.dart b/dev/bots/utils.dart index be8ab9b028..3ed0958ff0 100644 --- a/dev/bots/utils.dart +++ b/dev/bots/utils.dart @@ -342,6 +342,8 @@ Future runDartTest(String workingDirectory, { bool ensurePrecompiledTool = true, bool shuffleTests = true, bool collectMetrics = false, + List? tags, + bool runSkipped = false, }) async { int? cpus; final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml @@ -379,6 +381,10 @@ Future runDartTest(String workingDirectory, { '--coverage=$coverage', if (perTestTimeout != null) '--timeout=${perTestTimeout.inMilliseconds}ms', + if (runSkipped) + '--run-skipped', + if (tags != null) + ...tags.map((String t) => '--tags=$t'), if (testPaths != null) for (final String testPath in testPaths) testPath, diff --git a/packages/flutter_tools/test/flutter_build_apk.shard/README.md b/packages/flutter_tools/test/flutter_build_apk.shard/README.md deleted file mode 100644 index d5d85c11f4..0000000000 --- a/packages/flutter_tools/test/flutter_build_apk.shard/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# `flutter_build_apk.shard` - -Integration tests that debug why `flutter build apk` sometimes stalls on CI. diff --git a/packages/flutter_tools/test/flutter_build_apk.shard/flutter_build_apk_test.dart b/packages/flutter_tools/test/flutter_build_apk.shard/flutter_build_apk_test.dart deleted file mode 100644 index 8a72d191c7..0000000000 --- a/packages/flutter_tools/test/flutter_build_apk.shard/flutter_build_apk_test.dart +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:io' as io; - -import 'package:file/file.dart'; - -import '../integration.shard/test_utils.dart'; -import '../src/common.dart'; - -void main() { - final String flutterRoot = getFlutterRoot(); - final String flutterBin = fileSystem.path.join(flutterRoot, 'bin', 'flutter'); - - late Directory tmpDir; - - setUp(() { - tmpDir = fileSystem.systemTempDirectory.createTempSync(); - }); - - tearDown(() { - tryToDelete(tmpDir); - }); - - Future setGradleLoggingLevel( - String level, { - required Directory projectDir, - }) async { - // Open gradle.properties and append to it. - final Directory androidDir = projectDir.childDirectory('android'); - final File gradleDotProperties = androidDir.childFile('gradle.properties'); - final io.IOSink sink = gradleDotProperties.openWrite(mode: FileMode.append); - - sink.writeln('org.gradle.logging.level=$level'); - await sink.flush(); - await sink.close(); - - // For debugging, print the current output. - io.stderr.writeln('${gradleDotProperties.path}:'); - io.stderr.writeln(gradleDotProperties.readAsStringSync()); - } - - // Normally these tests should take about a minute, but sometimes for - // unknown reasons they can take 30m+ and timeout. The intent behind this loop - // is to get more information on what exactly is happening. - for (int i = 1; i <= 10; i++) { - test('flutter build apk | attempt $i of 10', () async { - final String package = 'flutter_build_apk_test_$i'; - - // Create a new Flutter app. - await expectLater( - processManager.run( - [ - flutterBin, - 'create', - package, - ], - workingDirectory: tmpDir.path, - ), - completion(const ProcessResultMatcher()), - reason: 'Should create a new blank Flutter project', - ); - - // Tweak verbosity of just gradle. - final Directory projectDir = tmpDir.childDirectory(package); - await setGradleLoggingLevel('debug', projectDir: projectDir); - - // Build the APK. - final List args = [ - flutterBin, - '--verbose', - 'build', - 'apk', - '--debug', - ]; - io.stderr.writeln('Running $args...'); - - final io.Process process = await processManager.start( - args, - workingDirectory: projectDir.path, - mode: io.ProcessStartMode.inheritStdio, - ); - await expectLater(process.exitCode, completion(0)); - }); - } -} diff --git a/packages/flutter_tools/test/integration.shard/android_e2e_api_test.dart b/packages/flutter_tools/test/integration.shard/android_e2e_api_test.dart index 73d11c688b..9c1c677f6a 100644 --- a/packages/flutter_tools/test/integration.shard/android_e2e_api_test.dart +++ b/packages/flutter_tools/test/integration.shard/android_e2e_api_test.dart @@ -2,6 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(matanlurey): Remove after debugging https://github.com/flutter/flutter/issues/159000. +@Tags(['flutter-build-apk']) +library; + import 'package:file/file.dart'; import 'package:flutter_tools/src/base/io.dart'; diff --git a/packages/flutter_tools/test/integration.shard/android_gradle_deprecated_plugin_apply_test.dart b/packages/flutter_tools/test/integration.shard/android_gradle_deprecated_plugin_apply_test.dart index 5964218eed..91bc50622a 100644 --- a/packages/flutter_tools/test/integration.shard/android_gradle_deprecated_plugin_apply_test.dart +++ b/packages/flutter_tools/test/integration.shard/android_gradle_deprecated_plugin_apply_test.dart @@ -2,6 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(matanlurey): Remove after debugging https://github.com/flutter/flutter/issues/159000. +@Tags(['flutter-build-apk']) +library; + import 'dart:convert'; import 'package:file/file.dart'; diff --git a/packages/flutter_tools/test/integration.shard/android_gradle_legacy_flutter_plugins_strings_in_comments_test.dart b/packages/flutter_tools/test/integration.shard/android_gradle_legacy_flutter_plugins_strings_in_comments_test.dart index d5d7a46d7c..041b3aa453 100644 --- a/packages/flutter_tools/test/integration.shard/android_gradle_legacy_flutter_plugins_strings_in_comments_test.dart +++ b/packages/flutter_tools/test/integration.shard/android_gradle_legacy_flutter_plugins_strings_in_comments_test.dart @@ -2,6 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(matanlurey): Remove after debugging https://github.com/flutter/flutter/issues/159000. +@Tags(['flutter-build-apk']) +library; + // This test can be removed once https://github.com/flutter/flutter/issues/155484 is resolved. import 'package:flutter_tools/src/base/file_system.dart'; diff --git a/packages/flutter_tools/test/integration.shard/android_gradle_print_build_variants_test.dart b/packages/flutter_tools/test/integration.shard/android_gradle_print_build_variants_test.dart index 085d3a5832..fa2c72359f 100644 --- a/packages/flutter_tools/test/integration.shard/android_gradle_print_build_variants_test.dart +++ b/packages/flutter_tools/test/integration.shard/android_gradle_print_build_variants_test.dart @@ -2,6 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(matanlurey): Remove after debugging https://github.com/flutter/flutter/issues/159000. +@Tags(['flutter-build-apk']) +library; + import 'dart:convert'; import 'package:collection/collection.dart'; diff --git a/packages/flutter_tools/test/integration.shard/android_plugin_example_app_build_test.dart b/packages/flutter_tools/test/integration.shard/android_plugin_example_app_build_test.dart index 755ef2df44..8bb28785fa 100644 --- a/packages/flutter_tools/test/integration.shard/android_plugin_example_app_build_test.dart +++ b/packages/flutter_tools/test/integration.shard/android_plugin_example_app_build_test.dart @@ -2,6 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(matanlurey): Remove after debugging https://github.com/flutter/flutter/issues/159000. +@Tags(['flutter-build-apk']) +library; + import 'package:file_testing/file_testing.dart'; import 'package:flutter_tools/src/base/file_system.dart'; import 'package:flutter_tools/src/base/io.dart'; diff --git a/packages/flutter_tools/test/integration.shard/flutter_build_with_compilation_error_test.dart b/packages/flutter_tools/test/integration.shard/flutter_build_with_compilation_error_test.dart index 07de04c3c1..c37ec4f66c 100644 --- a/packages/flutter_tools/test/integration.shard/flutter_build_with_compilation_error_test.dart +++ b/packages/flutter_tools/test/integration.shard/flutter_build_with_compilation_error_test.dart @@ -2,6 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(matanlurey): Remove after debugging https://github.com/flutter/flutter/issues/159000. +@Tags(['flutter-build-apk']) +library; + import 'package:file/file.dart'; import 'package:flutter_tools/src/base/io.dart'; diff --git a/packages/flutter_tools/test/integration.shard/isolated/native_assets_agp_version_test.dart b/packages/flutter_tools/test/integration.shard/isolated/native_assets_agp_version_test.dart index 25f4e2331e..41acde6839 100644 --- a/packages/flutter_tools/test/integration.shard/isolated/native_assets_agp_version_test.dart +++ b/packages/flutter_tools/test/integration.shard/isolated/native_assets_agp_version_test.dart @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(matanlurey): Remove after debugging https://github.com/flutter/flutter/issues/159000. +@Tags(['flutter-build-apk']) @Timeout(Duration(minutes: 10)) library; diff --git a/packages/flutter_tools/test/integration.shard/isolated/native_assets_test.dart b/packages/flutter_tools/test/integration.shard/isolated/native_assets_test.dart index c4fe81939a..572fb2959f 100644 --- a/packages/flutter_tools/test/integration.shard/isolated/native_assets_test.dart +++ b/packages/flutter_tools/test/integration.shard/isolated/native_assets_test.dart @@ -10,6 +10,8 @@ // contain the native assets mapping. // When doing a hot reload, this mapping must stay in place. +// TODO(matanlurey): Remove after debugging https://github.com/flutter/flutter/issues/159000. +@Tags(['flutter-build-apk']) @Timeout(Duration(minutes: 10)) library; @@ -224,6 +226,7 @@ void main() { // TODO(matanlurey): Debug why flutter build apk often timesout. // See https://github.com/flutter/flutter/issues/158560 for details. skip: buildSubcommand == 'apk' ? 'flutter build apk times out' : false, // Temporary workaround for https://github.com/flutter/flutter/issues/158560. + tags: ['flutter-build-apk'], ); } } diff --git a/packages/flutter_tools/test/integration.shard/isolated/native_assets_without_cbuild_assemble_test.dart b/packages/flutter_tools/test/integration.shard/isolated/native_assets_without_cbuild_assemble_test.dart index f80799d550..2e36b4f616 100644 --- a/packages/flutter_tools/test/integration.shard/isolated/native_assets_without_cbuild_assemble_test.dart +++ b/packages/flutter_tools/test/integration.shard/isolated/native_assets_without_cbuild_assemble_test.dart @@ -2,6 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(matanlurey): Remove after debugging https://github.com/flutter/flutter/issues/159000. +@Tags(['flutter-build-apk']) +library; + import 'dart:io' as io; import 'package:file/file.dart'; @@ -143,6 +147,7 @@ void main(List args) async { // TODO(matanlurey): Debug why flutter build apk often timesout. // See https://github.com/flutter/flutter/issues/158560 for details. skip: buildCommand == 'apk' ? 'flutter build apk times out' : false, // Temporary workaround for https://github.com/flutter/flutter/issues/158560. + tags: ['flutter-build-apk'], ); }