diff --git a/dev/devicelab/lib/tasks/web_benchmarks.dart b/dev/devicelab/lib/tasks/web_benchmarks.dart
index 749601b8b7..121de740ea 100644
--- a/dev/devicelab/lib/tasks/web_benchmarks.dart
+++ b/dev/devicelab/lib/tasks/web_benchmarks.dart
@@ -99,8 +99,13 @@ Future<TaskResult> runWebBenchmark({ @required bool useCanvasKit }) async {
         '--disable-translate',
       ];
 
-      final bool isUncalibratedSmokeTest =
-          io.Platform.environment['UNCALIBRATED_SMOKE_TEST'] == 'true';
+      // TODO(yjbanov): temporarily disables headful Chrome until we get
+      //                devicelab hardware that is able to run it. Our current
+      //                GCE VMs can only run in headless mode.
+      //                See: https://github.com/flutter/flutter/issues/50164
+      final bool isUncalibratedSmokeTest = io.Platform.environment['CALIBRATED'] != 'true';
+      // final bool isUncalibratedSmokeTest =
+      //     io.Platform.environment['UNCALIBRATED_SMOKE_TEST'] == 'true';
       if (isUncalibratedSmokeTest) {
         print('Running in headless mode because running on uncalibrated hardware.');
         args.add('--headless');