am 38495164: Merge "change mechanism to report performance results" into jb-mr2-dev
* commit '38495164caf6de15071ed3675dcea4ff423ef887':
change mechanism to report performance results
diff --git a/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/DeviceReportLog.java b/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/DeviceReportLog.java
index 9747ca5..43e93d3 100644
--- a/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/DeviceReportLog.java
+++ b/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/DeviceReportLog.java
@@ -16,10 +16,16 @@
package com.android.pts.util;
+import android.app.Instrumentation;
+import android.os.Bundle;
import android.util.Log;
+import java.lang.Exception;
+
public class DeviceReportLog extends ReportLog {
- private static final String TAG = "PtsReport";
+ private static final String TAG = "DevicePtsReport";
+ private static final String PTS_RESULT = "PTS_RESULT";
+ private static final int INST_STATUS_IN_PROGRESS = 2;
DeviceReportLog() {
mDepth = 4;
@@ -29,4 +35,11 @@
protected void printLog(String msg) {
Log.i(TAG, msg);
}
+
+ public void deliverReportToHost(Instrumentation instrumentation) {
+ Log.i(TAG, "deliverReportToHost");
+ Bundle output = new Bundle();
+ output.putString(PTS_RESULT, generateReport());
+ instrumentation.sendStatus(INST_STATUS_IN_PROGRESS, output);
+ }
}
diff --git a/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/DummyActivity.java b/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/DummyActivity.java
new file mode 100644
index 0000000..c3ceb9f
--- /dev/null
+++ b/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/DummyActivity.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.android.pts.util;
+
+import android.app.Activity;
+
+public class DummyActivity extends Activity {
+
+}
diff --git a/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/PtsActivityInstrumentationTestCase2.java b/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/PtsActivityInstrumentationTestCase2.java
index 0c9cdfa..a14be01 100644
--- a/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/PtsActivityInstrumentationTestCase2.java
+++ b/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/PtsActivityInstrumentationTestCase2.java
@@ -24,7 +24,7 @@
public class PtsActivityInstrumentationTestCase2<T extends Activity> extends
ActivityInstrumentationTestCase2<T> {
- private ReportLog mReportLog = new DeviceReportLog();
+ private DeviceReportLog mReportLog = new DeviceReportLog();
public PtsActivityInstrumentationTestCase2(Class<T> activityClass) {
super(activityClass);
@@ -36,7 +36,8 @@
@Override
protected void tearDown() throws Exception {
- mReportLog.throwReportToHost();
+ mReportLog.deliverReportToHost(getInstrumentation());
+ super.tearDown();
}
}
diff --git a/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/PtsAndroidTestCase.java b/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/PtsAndroidTestCase.java
index b847df7..6da7bdc 100644
--- a/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/PtsAndroidTestCase.java
+++ b/suite/pts/deviceTests/ptsutil/src/com/android/pts/util/PtsAndroidTestCase.java
@@ -17,18 +17,20 @@
package com.android.pts.util;
+import android.content.Context;
import android.test.AndroidTestCase;
-public class PtsAndroidTestCase extends AndroidTestCase {
-
- private ReportLog mReportLog = new DeviceReportLog();
-
- public ReportLog getReportLog() {
- return mReportLog;
+/**
+ * This class emulates AndroidTestCase, but internally it is ActivityInstrumentationTestCase2
+ * to access Instrumentation.
+ * DummyActivity is not supposed to be accessed.
+ */
+public class PtsAndroidTestCase extends PtsActivityInstrumentationTestCase2<DummyActivity> {
+ public PtsAndroidTestCase() {
+ super(DummyActivity.class);
}
- @Override
- protected void tearDown() throws Exception {
- mReportLog.throwReportToHost();
+ public Context getContext() {
+ return getInstrumentation().getContext();
}
}
diff --git a/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java b/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java
index dd7b834..636b194 100644
--- a/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java
+++ b/suite/pts/hostTests/bootup/src/com/android/pts/bootup/BootupTimeTest.java
@@ -17,6 +17,7 @@
package com.android.pts.bootup;
import android.cts.util.TimeoutReq;
+import com.android.pts.util.HostReportLog;
import com.android.pts.util.MeasureRun;
import com.android.pts.util.MeasureTime;
import com.android.pts.util.ResultType;
@@ -32,27 +33,14 @@
* Measure reboot-time using adb shell reboot
*/
public class BootupTimeTest extends DeviceTestCase {
-
- private ReportLog mReport = null;
// add some delay before each reboot
final static long SLEEP_BEFORE_REBOOT_TIME = 2 * 60 * 1000L;
final static int REBOOT_TIMEOUT_MS = 10 * 60 * 1000;
- @Override
- protected void setUp() throws Exception {
- mReport = new ReportLog();
- super.setUp();
- }
-
-
- @Override
- protected void tearDown() throws Exception {
- super.tearDown();
- mReport.throwReportToHost();
- }
-
@TimeoutReq(minutes = 30)
public void testBootupTime() throws Exception {
+ HostReportLog report =
+ new HostReportLog(getDevice().getSerialNumber(), ReportLog.getClassMethodNames());
final int NUMBER_REPEAT = 5;
double[] result = MeasureTime.measure(NUMBER_REPEAT, new MeasureRun() {
@Override
@@ -67,11 +55,12 @@
rebootDevice();
}
});
- mReport.printArray("bootup time", result, ResultType.LOWER_BETTER,
+ report.printArray("bootup time", result, ResultType.LOWER_BETTER,
ResultUnit.MS);
StatResult stat = Stat.getStat(result);
- mReport.printSummary("bootup time", stat.mAverage, ResultType.LOWER_BETTER,
+ report.printSummary("bootup time", stat.mAverage, ResultType.LOWER_BETTER,
ResultUnit.MS);
+ report.deliverReportToHost();
}
private void rebootDevice() throws DeviceNotAvailableException {
diff --git a/suite/pts/hostTests/ptshostutil/src/com/android/pts/ptsutil/LogcatLineReceiver.java b/suite/pts/hostTests/ptshostutil/src/com/android/pts/ptsutil/LogcatLineReceiver.java
deleted file mode 100644
index cb4980c..0000000
--- a/suite/pts/hostTests/ptshostutil/src/com/android/pts/ptsutil/LogcatLineReceiver.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.pts.ptsutil;
-
-import com.android.tradefed.device.ITestDevice;
-
-/**
- * class to handle logcat log per each line.
- * processALine is supposed to be overridden.
- */
-public class LogcatLineReceiver extends LogcatMonitor {
- public LogcatLineReceiver(ITestDevice device, String logcatFilter, long timeoutInSec) {
- super(device, logcatFilter, timeoutInSec);
- }
-
- /**
- * empty default implementation. Will be called whenever a line of log is received
- * @param line
- * @throws Exception
- */
- public void processALine(String line) throws Exception {
-
- }
-
- @Override
- public void addOutput(byte[] data, int offset, int length) {
- String lines = new String(data, offset, length);
- try {
- for (String line : lines.split("\n")) {
- processALine(line);
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-}
diff --git a/suite/pts/hostTests/ptshostutil/src/com/android/pts/ptsutil/LogcatMonitor.java b/suite/pts/hostTests/ptshostutil/src/com/android/pts/ptsutil/LogcatMonitor.java
deleted file mode 100644
index adc1340..0000000
--- a/suite/pts/hostTests/ptshostutil/src/com/android/pts/ptsutil/LogcatMonitor.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.pts.ptsutil;
-
-import com.android.ddmlib.IShellOutputReceiver;
-import com.android.tradefed.device.BackgroundDeviceAction;
-import com.android.tradefed.device.ITestDevice;
-
-import java.io.IOException;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-/**
- * class to monitor adb logcat asynchronously.
- * Unlike tradefed's LogcatReceiver, log is accumulated to a buffer, and the log is removed after
- * reading once.
- */
-public class LogcatMonitor implements IShellOutputReceiver {
- private static final String TAG = "LogcatMonitor";
- private LinkedBlockingQueue<String> mQ = new LinkedBlockingQueue<String>();
- private boolean mRunning = false;
- private long mTimeoutInSec;
- private BackgroundDeviceAction mDeviceAction;
- // sentinel to detect EOS
- private static final String EOS = "LOGCAT_MONITOR_EOS";
- private static final String LOGCAT_CMD = "logcat -v threadtime ";
-
- public LogcatMonitor(ITestDevice device, String logcatFilter, long timeoutInSec) {
- mTimeoutInSec = timeoutInSec;
- int logStartDelay = 0;
- mDeviceAction = new BackgroundDeviceAction(LOGCAT_CMD + logcatFilter, TAG, device,
- this, logStartDelay);
- }
-
- /** start monitoring log */
- public void start() {
- if (mRunning) {
- return;
- }
- clear();
- mDeviceAction.start();
- mRunning = true;
- }
-
- /** stop monitoring */
- public void stop() {
- mDeviceAction.cancel();
- mRunning = false;
- }
-
- /** clear all stored logs */
- public void clear() {
- mQ.clear();
- }
-
- /**
- * read a line of log. If there is no data stored, it can wait until
- * the timeout specified in the constructor.
- * @return null for time-out. Otherwise, a line of log is returned.
- * @throws IOException for EOS (logcat terminated for whatever reason)
- * @throws InterruptedException
- */
- public String getALine() throws IOException, InterruptedException {
- String line = mQ.poll(mTimeoutInSec, TimeUnit.SECONDS);
- if (line == EOS) {
- throw new IOException();
- }
- return line;
- }
-
- public boolean dataAvailable() {
- return !mQ.isEmpty();
- }
-
- @Override
- public void addOutput(byte[] data, int offset, int length) {
- String lines = new String(data, offset, length);
- for (String line : lines.split("\n")) {
- mQ.add(line);
- }
- }
-
- @Override
- public void flush() {
- mQ.add(EOS);
- mRunning = false;
- }
-
- @Override
- public boolean isCancelled() {
- return !mRunning;
- }
-}
diff --git a/suite/pts/hostTests/ptshostutil/src/com/android/pts/util/HostReportLog.java b/suite/pts/hostTests/ptshostutil/src/com/android/pts/util/HostReportLog.java
new file mode 100644
index 0000000..49f24fe
--- /dev/null
+++ b/suite/pts/hostTests/ptshostutil/src/com/android/pts/util/HostReportLog.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.pts.util;
+
+import com.android.cts.tradefed.result.PtsHostStore;
+
+
+/**
+ * ReportLog for host tests
+ * Note that setTestInfo should be set before throwing report
+ */
+public class HostReportLog extends ReportLog {
+ private String mDeviceSerial;
+ private String mClassMethodName;
+
+ /**
+ * @param deviceSerial serial number of the device
+ * @param classMethodName class name and method name of the test in class#method format.
+ * Note that ReportLog.getClassMethodNames() provide this.
+ */
+ public HostReportLog(String deviceSerial, String classMethodName) {
+ mDeviceSerial = deviceSerial;
+ mClassMethodName = classMethodName;
+ }
+
+ public void deliverReportToHost() {
+ PtsHostStore.storePtsResult(mDeviceSerial, mClassMethodName, generateReport());
+ }
+}
diff --git a/suite/pts/hostTests/uihost/src/com/android/pts/uihost/InstallTimeTest.java b/suite/pts/hostTests/uihost/src/com/android/pts/uihost/InstallTimeTest.java
index 056e0f4..55e9b40 100644
--- a/suite/pts/hostTests/uihost/src/com/android/pts/uihost/InstallTimeTest.java
+++ b/suite/pts/hostTests/uihost/src/com/android/pts/uihost/InstallTimeTest.java
@@ -17,6 +17,7 @@
package com.android.pts.uihost;
import com.android.cts.tradefed.build.CtsBuildHelper;
+import com.android.pts.util.HostReportLog;
import com.android.pts.util.MeasureRun;
import com.android.pts.util.MeasureTime;
import com.android.pts.util.ResultType;
@@ -38,7 +39,6 @@
public class InstallTimeTest extends DeviceTestCase implements IBuildReceiver {
private CtsBuildHelper mBuild;
private ITestDevice mDevice;
- private ReportLog mReport = null;
static final String PACKAGE = "com.replica.replicaisland";
static final String APK = "com.replica.replicaisland.apk";
@@ -51,19 +51,19 @@
@Override
protected void setUp() throws Exception {
super.setUp();
- mReport = new ReportLog();
mDevice = getDevice();
}
@Override
protected void tearDown() throws Exception {
- super.tearDown();
mDevice.uninstallPackage(PACKAGE);
- mReport.throwReportToHost();
+ super.tearDown();
}
public void testInstallTime() throws Exception {
+ HostReportLog report =
+ new HostReportLog(mDevice.getSerialNumber(), ReportLog.getClassMethodNames());
final int NUMBER_REPEAT = 10;
final CtsBuildHelper build = mBuild;
final ITestDevice device = mDevice;
@@ -78,11 +78,12 @@
device.installPackage(app, false);
}
});
- mReport.printArray("install time", result, ResultType.LOWER_BETTER,
+ report.printArray("install time", result, ResultType.LOWER_BETTER,
ResultUnit.MS);
StatResult stat = Stat.getStat(result);
- mReport.printSummary("install time", stat.mAverage, ResultType.LOWER_BETTER,
+ report.printSummary("install time", stat.mAverage, ResultType.LOWER_BETTER,
ResultUnit.MS);
+ report.deliverReportToHost();
}
}
diff --git a/suite/pts/hostTests/uihost/src/com/android/pts/uihost/TaskSwitchingTest.java b/suite/pts/hostTests/uihost/src/com/android/pts/uihost/TaskSwitchingTest.java
index 49d14a3..f1eb80a 100644
--- a/suite/pts/hostTests/uihost/src/com/android/pts/uihost/TaskSwitchingTest.java
+++ b/suite/pts/hostTests/uihost/src/com/android/pts/uihost/TaskSwitchingTest.java
@@ -19,14 +19,14 @@
import android.cts.util.TimeoutReq;
import com.android.cts.tradefed.build.CtsBuildHelper;
+import com.android.cts.tradefed.result.PtsReportUtil;
+import com.android.cts.tradefed.result.PtsHostStore;
import com.android.ddmlib.Log;
+import com.android.ddmlib.Log.LogLevel;
import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
import com.android.ddmlib.testrunner.TestIdentifier;
-import com.android.pts.util.MeasureRun;
-import com.android.pts.util.MeasureTime;
-import com.android.pts.util.PtsException;
-import com.android.pts.util.Stat;
-import com.android.pts.util.Stat.StatResult;
+import com.android.pts.util.HostReportLog;
+import com.android.pts.util.ReportLog;
import com.android.tradefed.build.IBuildInfo;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
@@ -40,6 +40,7 @@
import java.io.File;
import java.util.Map;
+
/**
* Measure time to taskswitching between two Apps: A & B
* Actual test is done in device, but this host side code installs all necessary APKs
@@ -50,6 +51,7 @@
private final static String CTS_RUNNER = "android.test.InstrumentationCtsTestRunner";
private CtsBuildHelper mBuild;
private ITestDevice mDevice;
+ private String mPtsReport = null;
static final String[] PACKAGES = {
"com.android.pts.taskswitching.control",
@@ -81,36 +83,36 @@
@Override
protected void tearDown() throws Exception {
- super.tearDown();
for (int i = 0; i < PACKAGES.length; i++) {
mDevice.uninstallPackage(PACKAGES[i]);
}
+ super.tearDown();
}
@TimeoutReq(minutes = 30)
public void testTaskswitching() throws Exception {
+ HostReportLog report =
+ new HostReportLog(mDevice.getSerialNumber(), ReportLog.getClassMethodNames());
RemoteAndroidTestRunner testRunner = new RemoteAndroidTestRunner(PACKAGES[0], CTS_RUNNER,
mDevice.getIDevice());
- CollectingTestListener listener = new CollectingTestListener();
+ LocalListener listener = new LocalListener();
mDevice.runInstrumentationTests(testRunner, listener);
TestRunResult result = listener.getCurrentRunResults();
if (result.isRunFailure()) {
fail(result.getRunFailureMessage());
}
- Map<TestIdentifier, TestResult> details = result.getTestResults();
- final String expectedException = "com.android.pts.util.PtsException";
- for (Map.Entry<TestIdentifier, TestResult> entry : details.entrySet()) {
- TestResult res = entry.getValue();
- String stackTrace = res.getStackTrace();
- if (stackTrace != null) {
- if (stackTrace.startsWith(expectedException)) {
- String[] lines = stackTrace.split("[\r\n]+");
- String msg = lines[0].substring(expectedException.length() + 1).trim();
- throw new PtsException(msg);
- }
- }
- }
- fail("no performance data");
+ assertNotNull("no performance data", mPtsReport);
+ PtsHostStore.storePtsResult(mDevice.getSerialNumber(),
+ ReportLog.getClassMethodNames(), mPtsReport);
+
}
+ public class LocalListener extends CollectingTestListener {
+ @Override
+ public void testEnded(TestIdentifier test, Map<String, String> testMetrics) {
+ // necessary as testMetrics passed from CollectingTestListerner is empty
+ mPtsReport = PtsReportUtil.getPtsResultFromMetrics(testMetrics);
+ super.testEnded(test, testMetrics);
+ }
+ }
}
diff --git a/suite/pts/lib/commonutil/Android.mk b/suite/pts/lib/commonutil/Android.mk
index 152f862..73df09b 100644
--- a/suite/pts/lib/commonutil/Android.mk
+++ b/suite/pts/lib/commonutil/Android.mk
@@ -28,7 +28,7 @@
include $(CLEAR_VARS)
-# only TmeoutReq annotation used from the libs/util, so add it here
+# only TimeoutReq annotation used from the libs/util, so add it here
LOCAL_SRC_FILES := \
$(call all-java-files-under, src) \
../../../../libs/util/src/android/cts/util/TimeoutReq.java
diff --git a/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java b/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java
index 8a5e624..63f5589 100644
--- a/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java
+++ b/suite/pts/lib/commonutil/src/com/android/pts/util/ReportLog.java
@@ -115,9 +115,10 @@
LOG_ELEM_SEPARATOR + unit.getXmlString() + LOG_ELEM_SEPARATOR + value;
}
- public void throwReportToHost() throws PtsException {
+ protected String generateReport() {
if ((mSummary == null) && mMessages.isEmpty()) {
- return;
+ // just return empty string
+ return "";
}
StringBuilder builder = new StringBuilder();
builder.append(mSummary);
@@ -132,7 +133,7 @@
}
mSummary = null;
mMessages.clear();
- throw new PtsException(builder.toString());
+ return builder.toString();
}
/**
@@ -177,7 +178,7 @@
}
/**
- * get classname.methodname from call stack of the current thread
+ * get classname#methodname from call stack of the current thread
*/
public static String getClassMethodNames() {
return getClassMethodNames(mDepth, false);
diff --git a/tools/tradefed-host/Android.mk b/tools/tradefed-host/Android.mk
index e8965ab..4a9e2fc 100644
--- a/tools/tradefed-host/Android.mk
+++ b/tools/tradefed-host/Android.mk
@@ -16,7 +16,8 @@
include $(CLEAR_VARS)
-# Only compile source java files in this lib.
+# suite/pts/hostTests/ptshostutil is treated specially
+# as it cannot be put into ptscommonutilhost due to dependency on cts-tradefed
LOCAL_SRC_FILES := \
$(call all-java-files-under, src) \
$(call all-java-files-under, ../../suite/pts/hostTests/ptshostutil/src)
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
index 2f62d71..9517d8e 100644
--- a/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java
@@ -99,9 +99,8 @@
private File mLogDir;
private String mSuiteName;
- private static final String PTS_PERFORMANCE_EXCEPTION = "com.android.pts.util.PtsException";
- private static final Pattern mPtsLogPattern = Pattern.compile(
- "com\\.android\\.pts\\.util\\.PtsException:\\s(.*)\\+\\+\\+\\+(.*)");
+ private static final Pattern mPtsLogPattern = Pattern.compile("(.*)\\+\\+\\+\\+(.*)");
+
public void setReportDir(File reportDir) {
mReportDir = reportDir;
}
@@ -226,21 +225,7 @@
*/
@Override
public void testFailed(TestFailure status, TestIdentifier test, String trace) {
- if (trace.startsWith(PTS_PERFORMANCE_EXCEPTION)) { //PTS result
- Test tst = mCurrentPkgResult.findTest(test);
- // this exception is always thrown as exception is thrown from tearDown.
- // Just ignore it.
- if (tst.getName().endsWith("testAndroidTestCaseSetupProperly")) {
- return;
- }
- Matcher m = mPtsLogPattern.matcher(trace);
- if (m.find()) {
- mCurrentPkgResult.reportPerformanceResult(test, CtsTestStatus.PASS, m.group(1),
- m.group(2));
- }
- } else {
- mCurrentPkgResult.reportTestFailure(test, CtsTestStatus.FAIL, trace);
- }
+ mCurrentPkgResult.reportTestFailure(test, CtsTestStatus.FAIL, trace);
}
/**
@@ -248,6 +233,7 @@
*/
@Override
public void testEnded(TestIdentifier test, Map<String, String> testMetrics) {
+ collectPtsResults(test, testMetrics);
mCurrentPkgResult.reportTestEnded(test);
Test result = mCurrentPkgResult.findTest(test);
String stack = result.getStackTrace() == null ? "" : "\n" + result.getStackTrace();
@@ -256,6 +242,31 @@
}
/**
+ * Collect Pts results for both device and host tests to the package result.
+ * @param test test ran
+ * @param testMetrics test metrics which can contain performance result for device tests
+ */
+ private void collectPtsResults(TestIdentifier test, Map<String, String> testMetrics) {
+ // device test can have performance results in testMetrics
+ String perfResult = PtsReportUtil.getPtsResultFromMetrics(testMetrics);
+ // host test should be checked in PtsHostStore.
+ if (perfResult == null) {
+ perfResult = PtsHostStore.removePtsResult(mDeviceSerial, test);
+ }
+ if (perfResult != null) {
+ // PTS result is passed in Summary++++Details format.
+ // Extract Summary and Details, and pass them.
+ Matcher m = mPtsLogPattern.matcher(perfResult);
+ if (m.find()) {
+ mCurrentPkgResult.reportPerformanceResult(test, CtsTestStatus.PASS, m.group(1),
+ m.group(2));
+ } else {
+ logResult("PTS Result unrecognizable:" + perfResult);
+ }
+ }
+ }
+
+ /**
* {@inheritDoc}
*/
@Override
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/PtsHostStore.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/PtsHostStore.java
new file mode 100644
index 0000000..5944c66
--- /dev/null
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/PtsHostStore.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.cts.tradefed.result;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.device.ITestDevice;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Utility class for storing Pts Results.
+ * This is necessary for host tests where test metrics cannot be passed.
+ */
+public class PtsHostStore {
+
+ // needs concurrent verion as there can be multiple client accessing this.
+ // But there is no additional protection for the same key as that should not happen.
+ private static final ConcurrentHashMap<String, String> mMap =
+ new ConcurrentHashMap<String, String>();
+
+ /**
+ * Stores PTS result. Existing result with the same key will be replaced.
+ * Note that key is generated in the form of device_serial#class#method name.
+ * So there should be no concurrent test for the same (serial, class, method).
+ * @param device
+ * @param test
+ * @param result PTS result string
+ */
+ public static void storePtsResult(String deviceSerial, String classMethodName, String result) {
+ mMap.put(generateTestKey(deviceSerial, classMethodName), result);
+ }
+
+ /**
+ * retrieves a PTS result for the given condition and remove it from the internal
+ * storage. If there is no result for the given condition, it will return null.
+ */
+ public static String removePtsResult(String deviceSerial, TestIdentifier test) {
+ return mMap.remove(generateTestKey(deviceSerial, test));
+ }
+
+ /**
+ * return test key in the form of device_serial#class_name#method_name
+ */
+ private static String generateTestKey(String deviceSerial, TestIdentifier test) {
+ return String.format("%s#%s", deviceSerial, test.toString());
+
+ }
+
+ /**
+ * return test key in the form of device_serial#class_name#method_name
+ */
+ private static String generateTestKey(String deviceSerial, String classMethodName) {
+ return String.format("%s#%s", deviceSerial, classMethodName);
+
+ }
+}
diff --git a/tools/tradefed-host/src/com/android/cts/tradefed/result/PtsReportUtil.java b/tools/tradefed-host/src/com/android/cts/tradefed/result/PtsReportUtil.java
new file mode 100644
index 0000000..2e3dab1
--- /dev/null
+++ b/tools/tradefed-host/src/com/android/cts/tradefed/result/PtsReportUtil.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.cts.tradefed.result;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.device.ITestDevice;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Static utility class for handling Pts Results.
+ */
+public class PtsReportUtil {
+ private static final String PTS_RESULT_KEY = "PTS_RESULT";
+
+ /**
+ * Utility method to extract PTS result from test metrics
+ * @param testMetrics
+ * @return result or null if not found
+ */
+ public static String getPtsResultFromMetrics(Map<String, String> testMetrics) {
+ for (Map.Entry<String, String> entry: testMetrics.entrySet()) {
+ if (PTS_RESULT_KEY.equals(entry.getKey())) {
+ return entry.getValue();
+ }
+ }
+ return null;
+ }
+}