Switch graphics_WebGLPerformance to telemetry.
And delete dependency on pyauto.
Also create separate deps for the content.
BUG=chromium:341333
TEST=test_that -b link 172.22.71.106 graphics_WebGLPerformance
CQ-DEPEND=185180
Change-Id: I32dfab349623ad8a7abd6dffa411f4a10a7b130c
Reviewed-on: https://chromium-review.googlesource.com/185292
Reviewed-by: Ilja Friedel <ihf@chromium.org>
Tested-by: Ilja Friedel <ihf@chromium.org>
Commit-Queue: Ilja Friedel <ihf@chromium.org>
diff --git a/client/deps/webgl_perf/common.py b/client/deps/webgl_perf/common.py
new file mode 100644
index 0000000..a0d813b
--- /dev/null
+++ b/client/deps/webgl_perf/common.py
@@ -0,0 +1,15 @@
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+
+dirname = os.path.dirname(sys.modules[__name__].__file__)
+# Load setup_modules from client_dir (two level up from current dir).
+client_dir = os.path.abspath(os.path.join(dirname, "..", ".."))
+sys.path.insert(0, client_dir)
+import setup_modules
+sys.path.pop(0)
+setup_modules.setup(base_path=client_dir,
+ root_module_name="autotest_lib.client")
diff --git a/client/deps/webgl_perf/control b/client/deps/webgl_perf/control
new file mode 100644
index 0000000..9968701
--- /dev/null
+++ b/client/deps/webgl_perf/control
@@ -0,0 +1,5 @@
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+job.setup_dep(['webgl_perf'])
diff --git a/client/site_tests/graphics_WebGLPerformance/favicon.ico b/client/deps/webgl_perf/files/favicon.ico
old mode 100755
new mode 100644
similarity index 100%
rename from client/site_tests/graphics_WebGLPerformance/favicon.ico
rename to client/deps/webgl_perf/files/favicon.ico
Binary files differ
diff --git a/client/site_tests/graphics_WebGLPerformance/scripts/update-tarball b/client/deps/webgl_perf/files/update-tarball
similarity index 100%
rename from client/site_tests/graphics_WebGLPerformance/scripts/update-tarball
rename to client/deps/webgl_perf/files/update-tarball
diff --git a/client/site_tests/graphics_WebGLPerformance/scripts/webgl-performance-0.0.1.tar.bz2 b/client/deps/webgl_perf/files/webgl-performance-0.0.1.tar.bz2
similarity index 100%
rename from client/site_tests/graphics_WebGLPerformance/scripts/webgl-performance-0.0.1.tar.bz2
rename to client/deps/webgl_perf/files/webgl-performance-0.0.1.tar.bz2
Binary files differ
diff --git a/client/deps/webgl_perf/files/webgl-performance-0.0.2.patch b/client/deps/webgl_perf/files/webgl-performance-0.0.2.patch
new file mode 100644
index 0000000..5f1ef39
--- /dev/null
+++ b/client/deps/webgl_perf/files/webgl-performance-0.0.2.patch
@@ -0,0 +1,66 @@
+From ea268329149aea6e21c2c0a571eafa0abd7ab9ea Mon Sep 17 00:00:00 2001
+From: "Ilja H. Friedel" <ihf@chromium.org>
+Date: Wed, 5 Feb 2014 22:05:22 -0800
+Subject: [PATCH] Patch webgl-performance-tests.html
+
+so it works with autotest graphics_WebGLPerformance.
+
+BUG=chromium:341333
+---
+ webgl-performance-tests.html | 18 +++++++++++++-----
+ 1 file changed, 13 insertions(+), 5 deletions(-)
+
+diff --git a/webgl-performance-tests.html b/webgl-performance-tests.html
+index a3533e7..026490e 100644
+--- a/webgl-performance-tests.html
++++ b/webgl-performance-tests.html
+@@ -2,6 +2,7 @@
+ <head>
+ <meta charset="utf-8">
+ </head>
++
+ <script>
+ var testPageURLs = [];
+ var testsRun = [];
+@@ -11,7 +12,10 @@
+ var sumOfResults = 0;
+ var sumOfLogResults = 0;
+ var numberOfResults = 0;
+-
++ // These variables are used to communicate with autotests.
++ var test_report = "Tests did not complete.";
++ var time_ms_geom_mean = -1.0;
++ var test_completed = false;
+
+ function startCurrentTestPage() {
+ if (currentPageIndex < testPageURLs.length) {
+@@ -22,7 +26,7 @@
+ testframe.src = testPageURLs[currentPageIndex];
+ } else {
+ testframe.parentNode.removeChild(testframe);
+- var t = "<b>WebGL Performance Regression Test Results</b><br/><br/>" +
++ var msg = "<b>WebGL Performance Regression Test Results</b><br/><br/>" +
+ "Keep in mind that these tests are not realistic workloads. These are not " +
+ "benchmarks aiming to compare browser or GPU performance. These are only useful " +
+ "to catch performance regressions in a given browser and system.<br/><br/>" +
+@@ -41,10 +45,14 @@
+ result = "<font color='blue'>Skipped</font>";
+ else
+ result = testsRun[i].testResult + " ms";
+- t += "<tr style='background-color:" + color + "'><td><a href='" + testPageURLs[i] + "'>" + testPageURLs[i] + "</a></td><td>" + result + "</td><td><i>" + testsRun[i].testDescription + "</i></td></tr>";
++ msg += "<tr style='background-color:" + color + "'><td><a href='" + testPageURLs[i] + "'>" + testPageURLs[i] + "</a></td><td>" + result + "</td><td><i>" + testsRun[i].testDescription + "</i></td></tr>";
+ }
+- t += "</table>";
+- resultsDiv.innerHTML = t;
++ msg += "</table>";
++ resultsDiv.innerHTML = msg;
++ // Fill out copy of results for autotest harness.
++ time_ms_geom_mean = Math.round(100 * Math.exp(sumOfLogResults / numberOfResults)) / 100;
++ test_report = msg;
++ test_completed = true;
+ }
+ }
+
+--
+1.9.0.rc1.175.g0b1dcb5
+
diff --git a/client/deps/webgl_perf/webgl_perf.py b/client/deps/webgl_perf/webgl_perf.py
new file mode 100755
index 0000000..584c168
--- /dev/null
+++ b/client/deps/webgl_perf/webgl_perf.py
@@ -0,0 +1,29 @@
+#!/usr/bin/python
+
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os, shutil
+from autotest_lib.client.bin import utils
+
+version = 1
+
+def setup(topdir):
+ """Unpack tarball to src/ and apply patch.
+ @param topdir: The directory of this deps.
+ """
+ tarball='webgl-performance-0.0.1.tar.bz2'
+ srcdir = os.path.join(topdir, 'src')
+ filesdir = os.path.join(topdir, 'files')
+ shutil.rmtree(srcdir, ignore_errors=True)
+ tarball_path = os.path.join(filesdir, tarball)
+ if not os.path.exists(srcdir):
+ os.mkdir(srcdir)
+ utils.extract_tarball_to_dir(tarball_path, srcdir)
+ os.chdir(srcdir)
+ utils.system('patch -p1 < ../files/webgl-performance-0.0.2.patch')
+ shutil.copy('../files/favicon.ico', srcdir)
+
+pwd = os.getcwd()
+utils.update_version(pwd + '/src', True, version, setup, pwd)
diff --git a/client/site_tests/graphics_WebGLPerformance/graphics_WebGLPerformance.py b/client/site_tests/graphics_WebGLPerformance/graphics_WebGLPerformance.py
index d440e1d..5baa136 100644
--- a/client/site_tests/graphics_WebGLPerformance/graphics_WebGLPerformance.py
+++ b/client/site_tests/graphics_WebGLPerformance/graphics_WebGLPerformance.py
@@ -1,88 +1,103 @@
-# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-import logging, os, shutil, time, urllib
+"""This is a client side WebGL performance test."""
+
+import logging, os, time
+
+from autotest_lib.client.bin import test
from autotest_lib.client.bin import utils
from autotest_lib.client.common_lib import error
-from autotest_lib.client.cros import cros_ui, cros_ui_test, graphics_ui_test
-from autotest_lib.client.cros import httpd
+from autotest_lib.client.common_lib.cros import chrome
-class graphics_WebGLPerformance(graphics_ui_test.GraphicsUITest):
+
+class graphics_WebGLPerformance(test.test):
+ """WebGL performance graphics test."""
version = 1
- def initialize(self, creds='$default'):
- self._test_url = 'http://localhost:8000/webgl-performance-tests.html'
- self._testServer = httpd.HTTPListener(8000, docroot=self.srcdir)
- self._testServer.run()
- graphics_ui_test.GraphicsUITest.initialize(self, creds,
- extra_chrome_flags=['--enable-webgl'])
+ def setup(self):
+ self.job.setup_dep(['webgl_perf'])
+ self.job.setup_dep(['graphics'])
- def setup(self, tarball='webgl-performance-0.0.1.tar.bz2'):
- shutil.rmtree(self.srcdir, ignore_errors=True)
- tarball_path = os.path.join(self.bindir, tarball)
- if not os.path.exists(self.srcdir):
- if not os.path.exists(tarball_path):
- utils.get_file(
- 'http://commondatastorage.googleapis.com/'
- 'chromeos-localmirror/distfiles/' + tarball,
- tarball_path)
- os.mkdir(self.srcdir)
- utils.extract_tarball_to_dir(tarball_path, self.srcdir)
- os.chdir(self.srcdir)
- utils.system('patch -p2 < ../webgl-performance-0.0.1.patch')
- shutil.copy('../favicon.ico', self.srcdir)
+ def initialize(self):
+ self.perf_keyval = {}
- def cleanup(self):
- self._testServer.stop()
- graphics_ui_test.GraphicsUITest.cleanup(self)
+ def poll_for_condition(self, tab, condition, error_msg):
+ """Waits until javascript condition is true.
- def run_once(self, timeout=600):
- # TODO(ihf): Remove when stable. For now we have to expect crashes.
- self.crash_blacklist.append('chrome')
- self.crash_blacklist.append('chromium')
+ @param tab: The tab the javascript/condition runs on.
+ @param condition: The javascript condition to evaluate.
+ @param error_msg: Test failure error string on timeout.
+ """
+ utils.poll_for_condition(
+ lambda: tab.EvaluateJavaScript(condition),
+ exception=error.TestError(error_msg),
+ timeout=self.test_duration_secs,
+ sleep_interval=1)
- latch = self._testServer.add_wait_url('/WebGL/results')
- # Loading the url might take longer than pyauto automation timeout.
- # Temporarily increment pyauto timeout.
- pyauto_timeout_changer = self.pyauto.ActionTimeoutChanger(
- self.pyauto, timeout * 1000)
- logging.info('Going to %s' % self._test_url)
- # TODO(ihf): Use the pyauto perf mechanisms to calm the system down.
- utils.system('sync')
- time.sleep(10.0)
- self.pyauto.NavigateToURL(self._test_url)
- del pyauto_timeout_changer
- latch.wait(timeout)
+ def run_performance_test(self, browser, test_url):
+ """Runs the performance test from the given url.
- if not latch.is_set():
- raise error.TestFail('Timeout after ' + str(timeout) +
- ' seconds - never received callback from browser.')
+ @param browser: The Browser object to run the test with.
+ @param test_url: The URL to the performance test site.
+ """
+ # Wait 5 seconds for the system to stabilize.
+ # TODO(ihf): Add a function that waits for low system load.
+ time.sleep(5)
- # Receive data from webgl-performance-tests.html::postFinalResults.
- results = self._testServer.get_form_entries()
- time_ms_geom_mean = float(results['time_ms_geom_mean'])
+ # Kick off test.
+ tab = browser.tabs.New()
+ tab.Navigate(test_url)
+ tab.Activate()
+
+ # Wait for test completion.
+ self.poll_for_condition(tab, 'time_ms_geom_mean > 0.0',
+ 'Timed out running the test.')
+
+ # Get the geometric mean of individual runtimes.
+ time_ms_geom_mean = tab.EvaluateJavaScript(
+ 'time_ms_geom_mean')
+ logging.info('WebGLPerformance: time_ms_geom_mean = %f',
+ time_ms_geom_mean)
# Output numbers for plotting by harness.
keyvals = {}
keyvals['time_ms_geom_mean'] = time_ms_geom_mean
- logging.info('WebGLPerformance: time_ms_geom_mean = %f'\
- % time_ms_geom_mean)
self.write_perf_keyval(keyvals)
- # TODO(ihf): Switch this test to Telemetry (in cros_ui_test.py) so that
- # the numbers actually make it to the perf dashboard.
self.output_perf_value(description='time_geom_mean',
value=time_ms_geom_mean, units='ms',
higher_is_better=False)
- # Write transmitted summary to graphics_WebGLPerformance/summary.html.
- summary = urllib.unquote_plus(results['summary'])
- logging.info('\n' + summary)
+ # Get a copy of the test report.
+ test_report = tab.EvaluateJavaScript('test_report')
results_path = os.path.join(self.bindir,
- "../../results/default/graphics_WebGLPerformance/summary.html")
+ "../../results/default/graphics_WebGLPerformance/test_report.html")
f = open(results_path, 'w+')
- f.write(summary)
+ f.write(test_report)
f.close()
- # Allow somebody to take a look at the screen.
- time.sleep(10.0)
+
+ tab.Close()
+
+ def run_once(self, test_duration_secs=600, fullscreen=True):
+ """Finds a brower with telemetry, and run the test.
+
+ @param test_duration_secs: The test duration in seconds.
+ @param fullscreen: Whether to run the test in fullscreen.
+ """
+ self.test_duration_secs = test_duration_secs
+
+ ext_paths = []
+ if fullscreen:
+ ext_paths.append(
+ os.path.join(self.autodir, 'deps', 'graphics',
+ 'graphics_test_extension'))
+
+ with chrome.Chrome(extension_paths=ext_paths) as cr:
+ websrc_dir = os.path.join(self.autodir, 'deps', 'webgl_perf', 'src')
+ if not cr.browser.SetHTTPServerDirectories(websrc_dir):
+ raise error.TestError('Unable to start HTTP server')
+ test_url = cr.browser.http_server.UrlOf(
+ os.path.join(websrc_dir, 'webgl-performance-tests.html'))
+ self.run_performance_test(cr.browser, test_url)
diff --git a/client/site_tests/graphics_WebGLPerformance/webgl-performance-0.0.1.patch b/client/site_tests/graphics_WebGLPerformance/webgl-performance-0.0.1.patch
deleted file mode 100644
index 6aadc31..0000000
--- a/client/site_tests/graphics_WebGLPerformance/webgl-performance-0.0.1.patch
+++ /dev/null
@@ -1,32 +0,0 @@
---- scripts/webgl-perf-tests/webgl-performance-tests.html 2012-03-29 19:54:37.000000000
-+++ scripts/webgl-perf-tests/webgl-performance-tests.html 2012-04-10 18:11:21.000000000
-@@ -42,13 +42,28 @@
- else
- result = testsRun[i].testResult + " ms";
- t += "<tr style='background-color:" + color + "'><td><a href='" + testPageURLs[i] + "'>" + testPageURLs[i] + "</a></td><td>" + result + "</td><td><i>" + testsRun[i].testDescription + "</i></td></tr>";
- }
- t += "</table>";
- resultsDiv.innerHTML = t;
-+ var msg = "&time_ms_geom_mean=";
-+ msg += (Math.round(100 * Math.exp(sumOfLogResults / numberOfResults)) / 100);
-+ msg += "&summary=" + escape(t);
-+ postFinalResults(msg);
- }
-+ }
-+
-+ // send results to autotest graphics_WebGLConformance.py
-+ postFinalResults = function(msg) {
-+ var xhr = new XMLHttpRequest();
-+ xhr.open('POST', 'http://localhost:8000/WebGL/results', false);
-+ xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
-+ xhr.onreadystatechange = function() {
-+ if (xhr.readyState != 4) { return; }
-+ }
-+ xhr.send(msg);
- }
-
- function messageListener(e) {
- if (e.data.testDescription) {
- testsRun.push(e.data);
- if (e.data.testResult) {
-