From 3d14b06ca965c7435e4c9f67407b204441116135 Mon Sep 17 00:00:00 2001 From: Adam Barth Date: Tue, 4 Nov 2014 16:45:43 -0800 Subject: [PATCH] Add a sketch of a test_perf script This script isn't quite working, but it's valuable to checkpoint the work at this point. I'm coordinating with the chromeperf.appspot.com folks to get the server to accept the data. R=eseidel@chromium.org Review URL: https://codereview.chromium.org/702603005 --- .../{simple-blocks.html => simple-blocks.sky} | 0 .../flutter/benchmarks/resources/runner.sky | 5 +- engine/src/flutter/tools/sky_server | 16 +- engine/src/flutter/tools/skydb | 21 +-- engine/src/flutter/tools/skypy/__init__.py | 3 + engine/src/flutter/tools/skypy/paths.py | 14 ++ engine/src/flutter/tools/test_perf | 138 +++++++++++++----- engine/src/flutter/tools/tester/BUILD.gn | 2 - .../src/flutter/tools/tester/test_harness.cc | 49 ------- .../src/flutter/tools/tester/test_harness.h | 39 ----- engine/src/flutter/tools/tester/tester.cc | 53 ++++++- 11 files changed, 183 insertions(+), 157 deletions(-) rename engine/src/flutter/benchmarks/layout/{simple-blocks.html => simple-blocks.sky} (100%) create mode 100644 engine/src/flutter/tools/skypy/__init__.py create mode 100644 engine/src/flutter/tools/skypy/paths.py delete mode 100644 engine/src/flutter/tools/tester/test_harness.cc delete mode 100644 engine/src/flutter/tools/tester/test_harness.h diff --git a/engine/src/flutter/benchmarks/layout/simple-blocks.html b/engine/src/flutter/benchmarks/layout/simple-blocks.sky similarity index 100% rename from engine/src/flutter/benchmarks/layout/simple-blocks.html rename to engine/src/flutter/benchmarks/layout/simple-blocks.sky diff --git a/engine/src/flutter/benchmarks/resources/runner.sky b/engine/src/flutter/benchmarks/resources/runner.sky index e307c0d273..8497a612db 100644 --- a/engine/src/flutter/benchmarks/resources/runner.sky +++ b/engine/src/flutter/benchmarks/resources/runner.sky @@ -4,14 +4,14 @@ function PerfRunner(options) { this.iterationsRemaining_ = options.iterations || 10; this.results_ = []; this.setup_ = options.setup; + this.logLines_ = []; } PerfRunner.prototype.log = function(line) { - console.log(line); + this.logLines_.push(line); }; PerfRunner.prototype.recordResult = function(result) { - console.log(result); this.results_.push(result); }; @@ -82,6 +82,7 @@ PerfRunner.prototype.logStatistics = function(title) { PerfRunner.prototype.finish = function () { this.logStatistics("Time:"); + internals.notifyTestComplete(this.logLines_.join('\n')); } module.exports = PerfRunner; diff --git a/engine/src/flutter/tools/sky_server b/engine/src/flutter/tools/sky_server index 31ea77a1ae..be4e0604d8 100755 --- a/engine/src/flutter/tools/sky_server +++ b/engine/src/flutter/tools/sky_server @@ -8,15 +8,7 @@ import cherrypy import json import os import staticdirindex - - -BUILD_DIRECTORY = 'out' -CONFIG_DIRECTORY = 'Debug' -SRC_ROOT = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir, - os.pardir)) -SKY_ROOT = os.path.join(SRC_ROOT, 'sky') -GEN_ROOT = os.path.join(SRC_ROOT, BUILD_DIRECTORY, CONFIG_DIRECTORY, 'gen') - +import skypy.paths as paths def skydir(section="", dir="", path="", **kwargs): if cherrypy.request.params.get('format') is None: @@ -63,15 +55,15 @@ def main(): }, '/mojo/public': { 'tools.staticdir.on': True, - 'tools.staticdir.dir': os.path.join(GEN_ROOT, 'mojo', 'public'), + 'tools.staticdir.dir': os.path.join(paths.GEN_ROOT, 'mojo', 'public'), }, '/mojo/services': { 'tools.staticdir.on': True, - 'tools.staticdir.dir': os.path.join(GEN_ROOT, 'mojo', 'services'), + 'tools.staticdir.dir': os.path.join(paths.GEN_ROOT, 'mojo', 'services'), }, '/sky/services': { 'tools.staticdir.on': True, - 'tools.staticdir.dir': os.path.join(GEN_ROOT, 'sky', 'services'), + 'tools.staticdir.dir': os.path.join(paths.GEN_ROOT, 'sky', 'services'), }, } cherrypy.quickstart(config=config) diff --git a/engine/src/flutter/tools/skydb b/engine/src/flutter/tools/skydb index 98e8c54f5e..ce428d80f6 100755 --- a/engine/src/flutter/tools/skydb +++ b/engine/src/flutter/tools/skydb @@ -4,27 +4,22 @@ # found in the LICENSE file. import argparse +import logging import os +import skypy.paths as paths +import socket; import subprocess import sys import urlparse -import logging -import socket; -OUT_DIR = 'out' -CONFIG_NAME = 'Debug' -SKY_TOOLS_DIRECTORY = os.path.abspath(os.path.join(__file__, os.pardir)) -MOJO_ROOT = os.path.abspath(os.path.join(SKY_TOOLS_DIRECTORY, os.pardir, - os.pardir)) -MOJO_SHELL_PATH = os.path.join(MOJO_ROOT, OUT_DIR, CONFIG_NAME, 'mojo_shell') - SUPPORTED_MIME_TYPES = [ 'text/html', 'text/sky', 'text/plain', ] + class SkyDebugger(object): def __init__(self): self._sky_server = None @@ -38,8 +33,8 @@ class SkyDebugger(object): HTTP_PORT = 9999 path = os.path.abspath(path) - if os.path.commonprefix([path, MOJO_ROOT]) == MOJO_ROOT: - server_root = MOJO_ROOT + if os.path.commonprefix([path, paths.SRC_ROOT]) == paths.SRC_ROOT: + server_root = paths.SRC_ROOT else: server_root = os.path.dirname(path) logging.warn( @@ -53,7 +48,7 @@ class SkyDebugger(object): HTTP_PORT) else: server_command = [ - os.path.join(SKY_TOOLS_DIRECTORY, 'sky_server'), + os.path.join(paths.SKY_TOOLS_DIRECTORY, 'sky_server'), server_root, str(HTTP_PORT), ] @@ -72,7 +67,7 @@ class SkyDebugger(object): content_handlers = ['%s,%s' % (mime_type, 'mojo://sky_viewer/') for mime_type in SUPPORTED_MIME_TYPES] shell_command = [ - MOJO_SHELL_PATH, + paths.MOJO_SHELL_PATH, '--v=1', '--content-handlers=%s' % ','.join(content_handlers), '--url-mappings=mojo:window_manager=mojo:sky_debugger', diff --git a/engine/src/flutter/tools/skypy/__init__.py b/engine/src/flutter/tools/skypy/__init__.py new file mode 100644 index 0000000000..4d6aabb953 --- /dev/null +++ b/engine/src/flutter/tools/skypy/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/engine/src/flutter/tools/skypy/paths.py b/engine/src/flutter/tools/skypy/paths.py new file mode 100644 index 0000000000..51bddc7d62 --- /dev/null +++ b/engine/src/flutter/tools/skypy/paths.py @@ -0,0 +1,14 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os + +BUILD_DIRECTORY = 'out' +CONFIG_DIRECTORY = 'Debug' +SRC_ROOT = os.path.abspath(os.path.join(__file__, + os.pardir, os.pardir, os.pardir, os.pardir)) +SKY_ROOT = os.path.join(SRC_ROOT, 'sky') +GEN_ROOT = os.path.join(SRC_ROOT, BUILD_DIRECTORY, CONFIG_DIRECTORY, 'gen') +SKY_TOOLS_DIRECTORY = os.path.join(SRC_ROOT, 'sky', 'tools') +MOJO_SHELL_PATH = os.path.join(SRC_ROOT, BUILD_DIRECTORY, CONFIG_DIRECTORY, 'mojo_shell') diff --git a/engine/src/flutter/tools/test_perf b/engine/src/flutter/tools/test_perf index 1bf8ec8627..70186b20f1 100755 --- a/engine/src/flutter/tools/test_perf +++ b/engine/src/flutter/tools/test_perf @@ -1,40 +1,110 @@ #!/usr/bin/env python -# Copyright (C) 2012 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. -"""Run performance tests.""" +import os +import re +import skypy.paths as paths +import subprocess +import requests -import logging -import sys -from webkitpy.performance_tests.perftestsrunner import PerfTestsRunner +SUPPORTED_MIME_TYPES = [ + 'text/html', + 'text/sky', + 'text/plain', +] +HTTP_PORT = 9999 -if '__main__' == __name__: - logging.basicConfig(level=logging.INFO, format="%(message)s") +DASHBOARD_URL = 'https://chromeperf.appspot.com/add_point' - sys.exit(PerfTestsRunner().run()) + +def sky_tester_command(url): + content_handlers = ['%s,%s' % (mime_type, 'mojo://sky_viewer/') + for mime_type in SUPPORTED_MIME_TYPES] + return [ + paths.MOJO_SHELL_PATH, + '--args-for=mojo://native_viewport_service/ --use-headless-config --use-osmesa', + '--args-for=mojo://window_manager/ %s' % url, + '--content-handlers=%s' % ','.join(content_handlers), + '--url-mappings=mojo:window_manager=mojo://sky_tester/', + 'mojo:window_manager', + ] + + +def start_sky_server(port): + return subprocess.Popen([ + os.path.join(paths.SKY_TOOLS_DIRECTORY, 'sky_server'), + paths.SRC_ROOT, + str(port), + ]) + + +def values_from_output(output): + # Parse out the raw values from the PerfRunner output: + # values 90, 89, 93 ms + # We'll probably need a fancier parser at some point. + match = re.search(r'values (.+) ms', output, flags=re.MULTILINE) + return map(float, match.group(1).split(', ')) + + +def create_json_blob(values): + revision = subprocess.check_output(["git", "show-ref", "HEAD", "-s"]).strip() + return { + "master": "master.mojo.perf", + "bot": "sky-release", + "point_id": 123456, # FIXME: We need to generate a monotonicly increasing number somehow. + "versions": { + "mojo": revision + }, + "chart_data": { + "format_version": "1.0", + "benchmark_name": "layout.simple-blocks", + "charts": { + "warm_times": { + "traces": { + "layout.simple-blocks": { + "type": "list_of_scalar_values", + "values": values, + }, + } + } + } + } + } + + +def send_json_to_dashboard(json): + requests.post(DASHBOARD_URL, params={ 'data': json }) + + +class PerfHarness(object): + def __init__(self): + self._sky_server = None + + def _start_server(self): + self._sky_server = start_sky_server(HTTP_PORT) + + def main(self): + test = 'http://localhost:9999/sky/benchmarks/layout/simple-blocks.sky' + + self._start_server() + output = subprocess.check_output(sky_tester_command(test)) + values = values_from_output(output) + json = create_json_blob(values) + send_json_to_dashboard(json) + + def shutdown(self): + if self._sky_server: + self._sky_server.terminate() + + +if __name__ == '__main__': + harness = PerfHarness() + try: + harness.main() + except (KeyboardInterrupt, SystemExit): + pass + finally: + harness.shutdown() diff --git a/engine/src/flutter/tools/tester/BUILD.gn b/engine/src/flutter/tools/tester/BUILD.gn index 1d6c7e3016..ae89606ec9 100644 --- a/engine/src/flutter/tools/tester/BUILD.gn +++ b/engine/src/flutter/tools/tester/BUILD.gn @@ -6,8 +6,6 @@ shared_library("tester") { output_name = "sky_tester" sources = [ - "test_harness.cc", - "test_harness.h", "test_observer_impl.cc", "test_observer_impl.h", "test_runner.cc", diff --git a/engine/src/flutter/tools/tester/test_harness.cc b/engine/src/flutter/tools/tester/test_harness.cc deleted file mode 100644 index 82d4fb9f16..0000000000 --- a/engine/src/flutter/tools/tester/test_harness.cc +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "sky/tools/tester/test_harness.h" - -#include "base/bind.h" -#include "base/message_loop/message_loop.h" -#include - -namespace sky { -namespace tester { -namespace { - -std::string WaitForURL() { - std::string url; - std::cin >> url; - return url; -} - -} // namespace - -TestHarness::TestHarness(mojo::View* container) - : container_(container), - weak_ptr_factory_(this) { - std::cout << "#READY\n"; - std::cout.flush(); -} - -TestHarness::~TestHarness() { -} - -void TestHarness::ScheduleRun() { - base::MessageLoop::current()->PostTask(FROM_HERE, - base::Bind(&TestHarness::Run, weak_ptr_factory_.GetWeakPtr())); -} - -void TestHarness::Run() { - DCHECK(!test_runner_); - test_runner_.reset(new TestRunner(this, container_, WaitForURL())); -} - -void TestHarness::OnTestComplete() { - test_runner_.reset(); - ScheduleRun(); -} - -} // namespace tester -} // namespace sky diff --git a/engine/src/flutter/tools/tester/test_harness.h b/engine/src/flutter/tools/tester/test_harness.h deleted file mode 100644 index 4c52bf0ce7..0000000000 --- a/engine/src/flutter/tools/tester/test_harness.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef SKY_TOOLS_TESTER_TEST_HARNESS_H_ -#define SKY_TOOLS_TESTER_TEST_HARNESS_H_ - -#include "base/memory/weak_ptr.h" -#include "sky/tools/tester/test_runner.h" - -namespace mojo{ -class View; -} - -namespace sky { -namespace tester { - -class TestHarness : public TestRunnerClient { - public: - explicit TestHarness(mojo::View* container); - virtual ~TestHarness(); - - void ScheduleRun(); - - private: - void Run(); - void OnTestComplete() override; - - mojo::View* container_; - scoped_ptr test_runner_; - base::WeakPtrFactory weak_ptr_factory_; - - MOJO_DISALLOW_COPY_AND_ASSIGN(TestHarness); -}; - -} // namespace tester -} // namespace sky - -#endif // SKY_TOOLS_TESTER_TEST_HARNESS_H_ diff --git a/engine/src/flutter/tools/tester/tester.cc b/engine/src/flutter/tools/tester/tester.cc index fd1f4ff45c..57815c6fb4 100644 --- a/engine/src/flutter/tools/tester/tester.cc +++ b/engine/src/flutter/tools/tester/tester.cc @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include +#include "base/bind.h" +#include "base/memory/weak_ptr.h" #include "mojo/application/application_runner_chromium.h" #include "mojo/public/c/system/main.h" #include "mojo/public/cpp/application/application_delegate.h" @@ -14,27 +17,41 @@ #include "mojo/services/public/interfaces/input_events/input_events.mojom.h" #include "mojo/services/window_manager/window_manager_app.h" #include "mojo/services/window_manager/window_manager_delegate.h" -#include "sky/tools/tester/test_harness.h" +#include "sky/tools/tester/test_runner.h" namespace sky { namespace tester { +namespace { + +std::string WaitForURL() { + std::string url; + std::cin >> url; + return url; +} + +} // namespace class SkyTester : public mojo::ApplicationDelegate, public mojo::ViewManagerDelegate, public mojo::WindowManagerDelegate, - public mojo::ViewObserver { + public mojo::ViewObserver, + public TestRunnerClient { public: SkyTester() : window_manager_app_(new mojo::WindowManagerApp(this, this)), view_manager_(NULL), root_(NULL), - content_(NULL) {} + content_(NULL), + weak_ptr_factory_(this) {} virtual ~SkyTester() {} private: // Overridden from mojo::ApplicationDelegate: virtual void Initialize(mojo::ApplicationImpl* impl) override { window_manager_app_->Initialize(impl); + + if (impl->args().size() >= 2) + url_from_args_ = impl->args()[1]; } virtual bool ConfigureIncomingConnection( mojo::ApplicationConnection* connection) override { @@ -56,8 +73,9 @@ class SkyTester : public mojo::ApplicationDelegate, content_->SetBounds(root_->bounds()); root_->AddChild(content_); - test_harness_.reset(new TestHarness(content_)); - test_harness_->ScheduleRun(); + std::cout << "#READY\n"; + std::cout.flush(); + ScheduleRun(); } // Overridden from WindowManagerDelegate: @@ -82,12 +100,35 @@ class SkyTester : public mojo::ApplicationDelegate, content_->SetBounds(new_bounds); } + void ScheduleRun() { + base::MessageLoop::current()->PostTask(FROM_HERE, + base::Bind(&SkyTester::Run, weak_ptr_factory_.GetWeakPtr())); + } + + void Run() { + DCHECK(!test_runner_); + std::string url = url_from_args_.length() ? url_from_args_ : WaitForURL(); + test_runner_.reset(new TestRunner(this, content_, url)); + } + + void OnTestComplete() override { + test_runner_.reset(); + if (url_from_args_.length()) + exit(0); + ScheduleRun(); + } + scoped_ptr window_manager_app_; + std::string url_from_args_; + mojo::ViewManager* view_manager_; mojo::View* root_; mojo::View* content_; - scoped_ptr test_harness_; + + scoped_ptr test_runner_; + + base::WeakPtrFactory weak_ptr_factory_; DISALLOW_COPY_AND_ASSIGN(SkyTester); };