e927e5488f
#flutter/flutter/127682 The plurality of android users are on api 33. We should be testing on our most used android api.
152 lines
4.1 KiB
Python
Executable File
152 lines
4.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Copyright 2013 The Flutter Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
import argparse
|
|
import glob
|
|
import re
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from compatibility_helper import byte_str_decode
|
|
|
|
if 'STORAGE_BUCKET' not in os.environ:
|
|
print('The GCP storage bucket must be provided as an environment variable.')
|
|
sys.exit(1)
|
|
BUCKET = os.environ['STORAGE_BUCKET']
|
|
|
|
if 'GCP_PROJECT' not in os.environ:
|
|
print('The GCP project must be provided as an environment variable.')
|
|
sys.exit(1)
|
|
PROJECT = os.environ['GCP_PROJECT']
|
|
|
|
script_dir = os.path.dirname(os.path.realpath(__file__))
|
|
buildroot_dir = os.path.abspath(os.path.join(script_dir, '..', '..'))
|
|
out_dir = os.path.join(buildroot_dir, 'out')
|
|
error_re = re.compile(r'[EF]/flutter.+')
|
|
|
|
|
|
def run_firebase_test(apk, results_dir):
|
|
# game-loop tests are meant for OpenGL apps.
|
|
# This type of test will give the application a handle to a file, and
|
|
# we'll write the timeline JSON to that file.
|
|
# See https://firebase.google.com/docs/test-lab/android/game-loop
|
|
# Pixel 5. As of this commit, this is a highly available device in FTL.
|
|
process = subprocess.Popen(
|
|
[
|
|
'gcloud',
|
|
'--project',
|
|
PROJECT,
|
|
'firebase',
|
|
'test',
|
|
'android',
|
|
'run',
|
|
'--type',
|
|
'game-loop',
|
|
'--app',
|
|
apk,
|
|
'--timeout',
|
|
'2m',
|
|
'--results-bucket',
|
|
BUCKET,
|
|
'--results-dir',
|
|
results_dir,
|
|
'--device',
|
|
'model=oriole,version=33',
|
|
],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
universal_newlines=True,
|
|
)
|
|
return process
|
|
|
|
|
|
def check_logcat(results_dir):
|
|
logcat = subprocess.check_output([
|
|
'gsutil', 'cat',
|
|
'%s/%s/*/logcat' % (BUCKET, results_dir)
|
|
])
|
|
logcat = byte_str_decode(logcat)
|
|
if not logcat:
|
|
sys.exit(1)
|
|
|
|
logcat_matches = error_re.findall(logcat)
|
|
if logcat_matches:
|
|
print('Errors in logcat:')
|
|
print(logcat_matches)
|
|
sys.exit(1)
|
|
|
|
|
|
def check_timeline(results_dir):
|
|
gsutil_du = subprocess.check_output([
|
|
'gsutil', 'du',
|
|
'%s/%s/*/game_loop_results/results_scenario_0.json' %
|
|
(BUCKET, results_dir)
|
|
])
|
|
gsutil_du = byte_str_decode(gsutil_du)
|
|
gsutil_du = gsutil_du.strip()
|
|
if gsutil_du == '0':
|
|
print('Failed to produce a timeline.')
|
|
sys.exit(1)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
'--variant',
|
|
dest='variant',
|
|
action='store',
|
|
default='android_profile_arm64',
|
|
help='The engine variant to run tests for.'
|
|
)
|
|
parser.add_argument(
|
|
'--build-id',
|
|
default=os.environ.get('SWARMING_TASK_ID', 'local_test'),
|
|
help='A unique build identifier for this test. Used to sort results in the GCS bucket.'
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
apks_dir = os.path.join(out_dir, args.variant, 'firebase_apks')
|
|
apks = glob.glob('%s/*.apk' % apks_dir)
|
|
|
|
if not apks:
|
|
print('No APKs found at %s' % apks_dir)
|
|
return 1
|
|
|
|
git_revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'],
|
|
cwd=script_dir)
|
|
git_revision = byte_str_decode(git_revision)
|
|
git_revision = git_revision.strip()
|
|
results = []
|
|
apk = None
|
|
for apk in apks:
|
|
results_dir = '%s/%s/%s' % (
|
|
os.path.basename(apk), git_revision, args.build_id
|
|
)
|
|
process = run_firebase_test(apk, results_dir)
|
|
results.append((results_dir, process))
|
|
|
|
for results_dir, process in results:
|
|
for line in iter(process.stdout.readline, ''):
|
|
print(line.strip())
|
|
return_code = process.wait()
|
|
if return_code != 0:
|
|
print('Firebase test failed with code: %s' % return_code)
|
|
sys.exit(return_code)
|
|
|
|
print('Checking logcat for %s' % results_dir)
|
|
check_logcat(results_dir)
|
|
# scenario_app produces a timeline, but the android image test does not.
|
|
if 'scenario' in apk:
|
|
print('Checking timeline for %s' % results_dir)
|
|
check_timeline(results_dir)
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|