From 59859df1265a20c3af309ed115936ac021cd0807 Mon Sep 17 00:00:00 2001 From: Jonah Williams Date: Fri, 4 Mar 2022 15:41:14 -0800 Subject: [PATCH] Add Windows performance benchmark (#99564) --- .ci.yaml | 11 +++ TESTOWNERS | 1 + ...ws_home_scroll_perf__timeline_summary.dart | 12 +++ dev/devicelab/lib/framework/devices.dart | 88 ++++++++++++++++++- dev/devicelab/lib/tasks/perf_tests.dart | 6 ++ .../flutter_tools/lib/src/commands/drive.dart | 5 +- .../commands.shard/hermetic/drive_test.dart | 34 ++++++- 7 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 dev/devicelab/bin/tasks/windows_home_scroll_perf__timeline_summary.dart diff --git a/.ci.yaml b/.ci.yaml index 442075fe38..0efd5a754a 100755 --- a/.ci.yaml +++ b/.ci.yaml @@ -4403,6 +4403,17 @@ targets: - bin/** - .ci.yaml + - name: Windows windows_home_scroll_perf__timeline_summary + recipe: devicelab/devicelab_drone + bringup: true + timeout: 60 + properties: + tags: > + ["devicelab","hostonly"] + task_name: windows_home_scroll_perf__timeline_summary + benchmark: "true" + scheduler: luci + - name: Windows_android basic_material_app_win__compile recipe: devicelab/devicelab_drone presubmit: false diff --git a/TESTOWNERS b/TESTOWNERS index 94740d85f2..3ddbd20d86 100644 --- a/TESTOWNERS +++ b/TESTOWNERS @@ -199,6 +199,7 @@ /dev/devicelab/bin/tasks/module_test_ios.dart @jmagman @flutter/tool /dev/devicelab/bin/tasks/plugin_lint_mac.dart @stuartmorgan @flutter/plugin /dev/devicelab/bin/tasks/entrypoint_dart_registrant.dart @aaclarke @flutter/plugin +/dev/devicelab/bin/tasks/windows_home_scroll_perf__timeline_summary.dart @jonahwilliams @flutter/engine ## Host only framework tests # Linux analyze diff --git a/dev/devicelab/bin/tasks/windows_home_scroll_perf__timeline_summary.dart b/dev/devicelab/bin/tasks/windows_home_scroll_perf__timeline_summary.dart new file mode 100644 index 0000000000..9f382fb0f7 --- /dev/null +++ b/dev/devicelab/bin/tasks/windows_home_scroll_perf__timeline_summary.dart @@ -0,0 +1,12 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.windows; + await task(createHomeScrollPerfTest()); +} diff --git a/dev/devicelab/lib/framework/devices.dart b/dev/devicelab/lib/framework/devices.dart index 6bbc66d4bf..e4b0b70873 100644 --- a/dev/devicelab/lib/framework/devices.dart +++ b/dev/devicelab/lib/framework/devices.dart @@ -52,7 +52,7 @@ String? _findMatchId(List idList, String idPattern) { DeviceDiscovery get devices => DeviceDiscovery(); /// Device operating system the test is configured to test. -enum DeviceOperatingSystem { android, androidArm, androidArm64 ,ios, fuchsia, fake } +enum DeviceOperatingSystem { android, androidArm, androidArm64 ,ios, fuchsia, fake, windows } /// Device OS to test on. DeviceOperatingSystem deviceOperatingSystem = DeviceOperatingSystem.android; @@ -71,6 +71,8 @@ abstract class DeviceDiscovery { return IosDeviceDiscovery(); case DeviceOperatingSystem.fuchsia: return FuchsiaDeviceDiscovery(); + case DeviceOperatingSystem.windows: + return WindowsDeviceDiscovery(); case DeviceOperatingSystem.fake: print('Looking for fake devices! You should not see this in release builds.'); return FakeDeviceDiscovery(); @@ -332,6 +334,41 @@ class AndroidDeviceDiscovery implements DeviceDiscovery { } } +class WindowsDeviceDiscovery implements DeviceDiscovery { + factory WindowsDeviceDiscovery() { + return _instance ??= WindowsDeviceDiscovery._(); + } + + WindowsDeviceDiscovery._(); + + static WindowsDeviceDiscovery? _instance; + + static const WindowsDevice _device = WindowsDevice(); + + @override + Future> checkDevices() async { + return {}; + } + + @override + Future chooseWorkingDevice() async { } + + @override + Future chooseWorkingDeviceById(String deviceId) async { } + + @override + Future> discoverDevices() async { + return ['windows']; + } + + @override + Future performPreflightTasks() async { } + + @override + Future get workingDevice async => _device; + +} + class FuchsiaDeviceDiscovery implements DeviceDiscovery { factory FuchsiaDeviceDiscovery() { return _instance ??= FuchsiaDeviceDiscovery._(); @@ -943,6 +980,55 @@ class IosDevice extends Device { } } +class WindowsDevice extends Device { + const WindowsDevice(); + + @override + String get deviceId => 'windows'; + + @override + Future> getMemoryStats(String packageName) async { + return {}; + } + + @override + Future home() async { } + + @override + Future isAsleep() async { + return false; + } + + @override + Future isAwake() async { + return true; + } + + @override + Stream get logcat => const Stream.empty(); + + @override + Future reboot() async { } + + @override + Future sendToSleep() async { } + + @override + Future stop(String packageName) async { } + + @override + Future tap(int x, int y) async { } + + @override + Future togglePower() async { } + + @override + Future unlock() async { } + + @override + Future wakeUp() async { } +} + /// Fuchsia device. class FuchsiaDevice extends Device { const FuchsiaDevice({ required this.deviceId }); diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index 0fa7bd30b9..7887ef8639 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -615,6 +615,7 @@ class StartupTest { ]); applicationBinaryPath = _findIosAppInBuildDirectory('$testDirectory/build/ios/iphoneos'); break; + case DeviceOperatingSystem.windows: case DeviceOperatingSystem.fuchsia: case DeviceOperatingSystem.fake: break; @@ -730,6 +731,7 @@ class DevtoolsStartupTest { ]); applicationBinaryPath = _findIosAppInBuildDirectory('$testDirectory/build/ios/iphoneos'); break; + case DeviceOperatingSystem.windows: case DeviceOperatingSystem.fuchsia: case DeviceOperatingSystem.fake: break; @@ -1307,6 +1309,8 @@ class CompileTest { if (reportPackageContentSizes) metrics.addAll(await getSizesFromApk(apkPath)); break; + case DeviceOperatingSystem.windows: + throw Exception('Unsupported option for Windows devices'); case DeviceOperatingSystem.fuchsia: throw Exception('Unsupported option for Fuchsia devices'); case DeviceOperatingSystem.fake: @@ -1343,6 +1347,8 @@ class CompileTest { options.insert(0, 'apk'); options.add('--target-platform=android-arm64'); break; + case DeviceOperatingSystem.windows: + throw Exception('Unsupported option for Windows devices'); case DeviceOperatingSystem.fuchsia: throw Exception('Unsupported option for Fuchsia devices'); case DeviceOperatingSystem.fake: diff --git a/packages/flutter_tools/lib/src/commands/drive.dart b/packages/flutter_tools/lib/src/commands/drive.dart index 2282875b18..f77b3d0afd 100644 --- a/packages/flutter_tools/lib/src/commands/drive.dart +++ b/packages/flutter_tools/lib/src/commands/drive.dart @@ -214,7 +214,7 @@ class DriveCommand extends RunCommandBase { throwToolExit(null); } if (screenshot != null && !device.supportsScreenshot) { - throwToolExit('Screenshot not supported for ${device.name}.'); + _logger.printError('Screenshot not supported for ${device.name}.'); } final bool web = device is WebServerDevice || device is ChromiumDevice; @@ -359,6 +359,9 @@ class DriveCommand extends RunCommandBase { } Future _takeScreenshot(Device device) async { + if (!device.supportsScreenshot) { + return; + } try { final Directory outputDirectory = _fileSystem.directory(screenshot) ..createSync(recursive: true); diff --git a/packages/flutter_tools/test/commands.shard/hermetic/drive_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/drive_test.dart index 37d39d35fe..b901cf05af 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/drive_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/drive_test.dart @@ -45,6 +45,38 @@ void main() { Cache.enableLocking(); }); + testUsingContext('warns if screenshot is not supported but continues test', () async { + final DriveCommand command = DriveCommand(fileSystem: fileSystem, logger: logger, platform: platform); + fileSystem.file('lib/main.dart').createSync(recursive: true); + fileSystem.file('test_driver/main_test.dart').createSync(recursive: true); + fileSystem.file('pubspec.yaml').createSync(); + fileSystem.directory('drive_screenshots').createSync(); + + final Device screenshotDevice = ThrowingScreenshotDevice() + ..supportsScreenshot = false; + fakeDeviceManager.devices = [screenshotDevice]; + + await expectLater(() => createTestCommandRunner(command).run( + [ + 'drive', + '--no-pub', + '-d', + screenshotDevice.id, + '--screenshot', + 'drive_screenshots', + ]), + throwsToolExit(message: 'cannot start app'), + ); + + expect(logger.errorText, contains('Screenshot not supported for FakeDevice')); + expect(logger.statusText, isEmpty); + }, overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => FakeProcessManager.any(), + Pub: () => FakePub(), + DeviceManager: () => fakeDeviceManager, + }); + testUsingContext('takes screenshot and rethrows on drive exception', () async { final DriveCommand command = DriveCommand(fileSystem: fileSystem, logger: logger, platform: platform); fileSystem.file('lib/main.dart').createSync(recursive: true); @@ -214,7 +246,7 @@ class ScreenshotDevice extends Fake implements Device { Future get targetPlatform async => TargetPlatform.android; @override - final bool supportsScreenshot = true; + bool supportsScreenshot = true; @override Future startApp(