Merge tag 'android-security-10.0.0_r53' into int/10/fp2
Android security 10.0.0 release 53
* tag 'android-security-10.0.0_r53':
Merge "Update the list of files to not copy at retry" am: 50dd6084b7 am: 2bcb57cd1f
Bump CTS to 10_R1
Remove the unused jar from the script
Change-Id: I86e70c36cd481f90ea5367d3cf0e0c5a1dbf24ec
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/build/CompatibilityBuildProvider.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/build/CompatibilityBuildProvider.java
index cfbfa1c..d84483e 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/build/CompatibilityBuildProvider.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/build/CompatibilityBuildProvider.java
@@ -32,7 +32,6 @@
import com.android.tradefed.testtype.IInvocationContextReceiver;
import com.android.tradefed.testtype.suite.TestSuiteInfo;
import com.android.tradefed.util.FileUtil;
-import com.android.tradefed.util.VersionParser;
import java.io.File;
import java.io.IOException;
@@ -311,12 +310,7 @@
* Return the SuiteInfo build number generated at build time. Exposed for testing.
*/
protected String getSuiteInfoBuildNumber() {
- String buildNumber = TestSuiteInfo.getInstance().getBuildNumber();
- String versionFile = VersionParser.fetchVersion();
- if (versionFile != null) {
- buildNumber = versionFile;
- }
- return buildNumber;
+ return TestSuiteInfo.getInstance().getBuildNumber();
}
/**
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
index 44939f3..1531d2f 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/ResultReporter.java
@@ -27,6 +27,7 @@
import com.android.compatibility.common.util.IModuleResult;
import com.android.compatibility.common.util.ITestResult;
import com.android.compatibility.common.util.InvocationResult;
+import com.android.compatibility.common.util.InvocationResult.RunHistory;
import com.android.compatibility.common.util.MetricsStore;
import com.android.compatibility.common.util.ReportLog;
import com.android.compatibility.common.util.ResultHandler;
@@ -34,8 +35,6 @@
import com.android.compatibility.common.util.TestStatus;
import com.android.ddmlib.Log.LogLevel;
import com.android.tradefed.build.IBuildInfo;
-import com.android.tradefed.config.IConfiguration;
-import com.android.tradefed.config.IConfigurationReceiver;
import com.android.tradefed.config.Option;
import com.android.tradefed.config.Option.Importance;
import com.android.tradefed.config.OptionClass;
@@ -64,6 +63,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.xml.XmlEscapers;
+import com.google.gson.Gson;
import org.xmlpull.v1.XmlPullParserException;
@@ -75,6 +75,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -89,7 +90,7 @@
*/
@OptionClass(alias="result-reporter")
public class ResultReporter implements ILogSaverListener, ITestInvocationListener,
- ITestSummaryListener, IShardableListener, IConfigurationReceiver {
+ ITestSummaryListener, IShardableListener {
public static final String INCLUDE_HTML_IN_ZIP = "html-in-zip";
private static final String UNKNOWN_DEVICE = "unknown_device";
@@ -97,6 +98,8 @@
private static final String CTS_PREFIX = "cts:";
private static final String BUILD_INFO = CTS_PREFIX + "build_";
private static final String LATEST_LINK_NAME = "latest";
+ /** Used to get run history from the test result of last run. */
+ private static final String RUN_HISTORY_KEY = "run_history";
public static final String BUILD_BRAND = "build_brand";
public static final String BUILD_DEVICE = "build_device";
@@ -188,9 +191,6 @@
// Elapsed time from invocation started to ended.
private long mElapsedTime;
- /** Invocation level configuration */
- private IConfiguration mConfiguration = null;
-
/**
* Default constructor.
*/
@@ -207,12 +207,6 @@
mMasterResultReporter = masterResultReporter;
}
- /** {@inheritDoc} */
- @Override
- public void setConfiguration(IConfiguration configuration) {
- mConfiguration = configuration;
- }
-
/**
* {@inheritDoc}
*/
@@ -533,6 +527,15 @@
}
}
+ /**
+ * Returns whether a report creation should be skipped.
+ */
+ protected boolean shouldSkipReportCreation() {
+ // This value is always false here for backwards compatibility.
+ // Extended classes have the option to override this.
+ return false;
+ }
+
private void finalizeResults() {
if (mFingerprintFailure) {
CLog.w("Failed the fingerprint check. Skip result reporting.");
@@ -565,6 +568,25 @@
String moduleProgress = String.format("%d of %d",
mResult.getModuleCompleteCount(), mResult.getModules().size());
+ // Get run history from the test result of last run and add the run history of the current
+ // run to it.
+ // TODO(b/137973382): avoid casting by move the method to interface level.
+ Collection<RunHistory> runHistories = ((InvocationResult) mResult).getRunHistories();
+ String runHistoryJSON = mResult.getInvocationInfo().get(RUN_HISTORY_KEY);
+ Gson gson = new Gson();
+ if (runHistoryJSON != null) {
+ RunHistory[] runHistoryArray = gson.fromJson(runHistoryJSON, RunHistory[].class);
+ Collections.addAll(runHistories, runHistoryArray);
+ }
+ RunHistory newRun = new RunHistory();
+ newRun.startTime = mResult.getStartTime();
+ newRun.endTime = newRun.startTime + mElapsedTime;
+ runHistories.add(newRun);
+ mResult.addInvocationInfo(RUN_HISTORY_KEY, gson.toJson(runHistories));
+
+ if (shouldSkipReportCreation()) {
+ return;
+ }
try {
// Zip the full test results directory.
@@ -750,7 +772,6 @@
fis = new FileInputStream(resultFile);
logFile = mLogSaver.saveLogData("log-result", LogDataType.XML, fis);
debug("Result XML URL: %s", logFile.getUrl());
- logReportFiles(mConfiguration, resultFile, resultFile.getName(), LogDataType.XML);
} catch (IOException ioe) {
CLog.e("[%s] error saving XML with log saver", mDeviceSerial);
CLog.e(ioe);
@@ -764,8 +785,6 @@
zipResultStream = new FileInputStream(zippedResults);
logFile = mLogSaver.saveLogData("results", LogDataType.ZIP, zipResultStream);
debug("Result zip URL: %s", logFile.getUrl());
- logReportFiles(
- mConfiguration, zippedResults, "results", LogDataType.ZIP);
} finally {
StreamUtil.close(zipResultStream);
}
@@ -1084,22 +1103,4 @@
private static String sanitizeXmlContent(String s) {
return XmlEscapers.xmlContentEscaper().escape(s);
}
-
- /** Re-log a result file to all reporters so they are aware of it. */
- private void logReportFiles(
- IConfiguration configuration, File resultFile, String dataName, LogDataType type) {
- if (configuration == null) {
- return;
- }
- List<ITestInvocationListener> listeners = configuration.getTestInvocationListeners();
- try (FileInputStreamSource source = new FileInputStreamSource(resultFile)) {
- for (ITestInvocationListener listener : listeners) {
- if (listener.equals(this)) {
- // Avoid logging agaisnt itself
- continue;
- }
- listener.testLog(dataName, type, source);
- }
- }
- }
}
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/suite/PreviousSessionFileCopier.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/suite/PreviousSessionFileCopier.java
index d0c7043..e8d1228 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/suite/PreviousSessionFileCopier.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/result/suite/PreviousSessionFileCopier.java
@@ -45,7 +45,8 @@
CertificationSuiteResultReporter.FAILURE_REPORT_NAME,
CertificationSuiteResultReporter.SUMMARY_FILE,
CertificationChecksumHelper.NAME,
- "diffs");
+ "diffs",
+ "proto");
private CompatibilityBuildHelper mBuildHelper;
private File mPreviousSessionDir = null;
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/ApkInstrumentationPreparer.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/ApkInstrumentationPreparer.java
index 80493f7..a1e5182 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/ApkInstrumentationPreparer.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/ApkInstrumentationPreparer.java
@@ -19,6 +19,8 @@
import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
import com.android.ddmlib.testrunner.TestResult.TestStatus;
import com.android.tradefed.build.IBuildInfo;
+import com.android.tradefed.config.IConfiguration;
+import com.android.tradefed.config.IConfigurationReceiver;
import com.android.tradefed.config.Option;
import com.android.tradefed.config.OptionClass;
import com.android.tradefed.device.DeviceNotAvailableException;
@@ -37,11 +39,10 @@
import java.io.FileNotFoundException;
import java.util.Map.Entry;
-/**
- * Target preparer that instruments an APK.
- */
-@OptionClass(alias="apk-instrumentation-preparer")
-public class ApkInstrumentationPreparer extends PreconditionPreparer implements ITargetCleaner {
+/** Target preparer that instruments an APK. */
+@OptionClass(alias = "apk-instrumentation-preparer")
+public class ApkInstrumentationPreparer extends PreconditionPreparer
+ implements IConfigurationReceiver, ITargetCleaner {
@Option(name = "apk", description = "Name of the apk to instrument", mandatory = true)
protected String mApkFileName = null;
@@ -59,6 +60,14 @@
@Option(name = "throw-error", description = "Whether to throw error for device test failure")
protected boolean mThrowError = true;
+ private IConfiguration mConfiguration = null;
+
+ /** {@inheritDoc} */
+ @Override
+ public void setConfiguration(IConfiguration configuration) {
+ mConfiguration = configuration;
+ }
+
/**
* {@inheritDoc}
*/
@@ -118,6 +127,7 @@
CLog.i("Instrumenting package: %s", mPackageName);
CollectingTestListener listener = new CollectingTestListener();
AndroidJUnitTest instrTest = new AndroidJUnitTest();
+ instrTest.setConfiguration(mConfiguration);
instrTest.setDevice(device);
instrTest.setInstallFile(apkFile);
instrTest.setPackageName(mPackageName);
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/BuildFingerPrintPreparer.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/BuildFingerPrintPreparer.java
index a96b76a..30e303b 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/BuildFingerPrintPreparer.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/BuildFingerPrintPreparer.java
@@ -54,17 +54,22 @@
if (!compare.equals(currentBuildFingerprint)) {
throw new TargetSetupError(
String.format(
- "Device build fingerprint must match %s. Found '%s' instead.",
+ "Device build fingerprint must match '%s'. Found '%s' instead.",
compare, currentBuildFingerprint),
device.getDeviceDescriptor());
}
if (mExpectedVendorFingerprint != null) {
String currentBuildVendorFingerprint =
device.getProperty(mVendorFingerprintProperty);
+ // Replace by empty string if null to do a proper comparison.
+ if (currentBuildVendorFingerprint == null) {
+ currentBuildVendorFingerprint = "";
+ }
if (!mExpectedVendorFingerprint.equals(currentBuildVendorFingerprint)) {
throw new TargetSetupError(
String.format(
- "Device vendor build fingerprint must match %s - found %s instead.",
+ "Device vendor build fingerprint must match '%s'. Found '%s' "
+ + "instead.",
mExpectedVendorFingerprint, currentBuildVendorFingerprint),
device.getDeviceDescriptor());
}
diff --git a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/BusinessLogicPreparer.java b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/BusinessLogicPreparer.java
index 48709fc..a65b95b 100644
--- a/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/BusinessLogicPreparer.java
+++ b/common/host-side/tradefed/src/com/android/compatibility/common/tradefed/targetprep/BusinessLogicPreparer.java
@@ -375,20 +375,32 @@
return extendedDeviceInfo;
}
File ediFile = null;
+ String[] fileAndKey = null;
try{
for (String ediEntry: requiredDeviceInfo) {
- String[] fileAndKey = ediEntry.split(":");
+ fileAndKey = ediEntry.split(":");
+ if (fileAndKey.length <= 1) {
+ CLog.e("Dynamic config Extended DeviceInfo key has problem.");
+ return new ArrayList<>();
+ }
ediFile = FileUtil
.findFile(deviceInfoPath, fileAndKey[0] + ".deviceinfo.json");
+ if (ediFile == null) {
+ CLog.e(
+ "Could not find Extended DeviceInfo JSON file: %s.",
+ deviceInfoPath + fileAndKey[0] + ".deviceinfo.json");
+ return new ArrayList<>();
+ }
String jsonString = FileUtil.readStringFromFile(ediFile);
JSONObject jsonObj = new JSONObject(jsonString);
String value = jsonObj.getString(fileAndKey[1]);
extendedDeviceInfo
.add(String.format("%s:%s:%s", fileAndKey[0], fileAndKey[1], value));
}
- }catch(JSONException | IOException e){
- CLog.e("Failed to read or parse Extended DeviceInfo JSON file: %s. Error: %s",
- ediFile.getAbsolutePath(), e);
+ }catch(JSONException | IOException | RuntimeException e){
+ CLog.e(
+ "Failed to read or parse Extended DeviceInfo JSON file: %s. Error: %s",
+ deviceInfoPath + fileAndKey[0] + ".deviceinfo.json", e);
return new ArrayList<>();
}
return extendedDeviceInfo;
diff --git a/common/host-side/tradefed/tests/run_tests.sh b/common/host-side/tradefed/tests/run_tests.sh
new file mode 100755
index 0000000..3cec79c
--- /dev/null
+++ b/common/host-side/tradefed/tests/run_tests.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HARNESS_DIR=$(dirname ${0})/../../../..
+source ${HARNESS_DIR}/test_defs.sh
+
+JARS="
+ compatibility-common-util-hostsidelib\
+ compatibility-common-util-tests\
+ compatibility-host-util\
+ compatibility-host-util-tests\
+ compatibility-mock-tradefed\
+ compatibility-tradefed-tests"
+
+run_tests "com.android.compatibility.common.tradefed.UnitTests" "${JARS}" "${@}"
+
diff --git a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java
index a179978..82c4d8f 100644
--- a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java
+++ b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/ResultReporterTest.java
@@ -549,4 +549,75 @@
TestStatus.PASS,
result1.getResultStatus());
}
+
+ /** Ensure that the run history of the current run is added to previous run history. */
+ public void testRetryWithRunHistory() throws Exception {
+ mReporter.invocationStarted(mContext);
+
+ // Set up IInvocationResult with existing results from previous session
+ mReporter.testRunStarted(ID, 2);
+ IInvocationResult invocationResult = mReporter.getResult();
+ IModuleResult moduleResult = invocationResult.getOrCreateModule(ID);
+ ICaseResult caseResult = moduleResult.getOrCreateResult(CLASS);
+ ITestResult testResult1 = caseResult.getOrCreateResult(METHOD_1);
+ testResult1.setResultStatus(TestStatus.PASS);
+ testResult1.setRetry(true);
+ ITestResult testResult2 = caseResult.getOrCreateResult(METHOD_2);
+ testResult2.setResultStatus(TestStatus.FAIL);
+ testResult2.setStackTrace(STACK_TRACE);
+ testResult2.setRetry(true);
+ // Set up IInvocationResult with the run history of previous runs.
+ invocationResult.addInvocationInfo(
+ "run_history", "[{\"startTime\":1,\"endTime\":2},{\"startTime\":3,\"endTime\":4}]");
+
+ // Flip results for the current session
+ TestDescription test1 = new TestDescription(CLASS, METHOD_1);
+ mReporter.testStarted(test1);
+ mReporter.testFailed(test1, STACK_TRACE);
+ mReporter.testEnded(test1, new HashMap<String, Metric>());
+ TestDescription test2 = new TestDescription(CLASS, METHOD_2);
+ mReporter.testStarted(test2);
+ mReporter.testEnded(test2, new HashMap<String, Metric>());
+
+ mReporter.testRunEnded(10, new HashMap<String, Metric>());
+ mReporter.invocationEnded(10);
+
+ // Verification that results have been overwritten.
+ IInvocationResult result = mReporter.getResult();
+ assertEquals("Expected 1 pass", 1, result.countResults(TestStatus.PASS));
+ assertEquals("Expected 1 failure", 1, result.countResults(TestStatus.FAIL));
+ List<IModuleResult> modules = result.getModules();
+ assertEquals("Expected 1 module", 1, modules.size());
+ IModuleResult module = modules.get(0);
+ List<ICaseResult> cases = module.getResults();
+ assertEquals("Expected 1 test case", 1, cases.size());
+ ICaseResult case1 = cases.get(0);
+ List<ITestResult> testResults = case1.getResults();
+ assertEquals("Expected 2 tests", 2, testResults.size());
+
+ long startTime = mReporter.getResult().getStartTime();
+ String expectedRunHistory =
+ String.format(
+ "[{\"startTime\":1,\"endTime\":2},"
+ + "{\"startTime\":3,\"endTime\":4},{\"startTime\":%d,\"endTime\":%d}]",
+ startTime, startTime + 10);
+ assertEquals(expectedRunHistory, invocationResult.getInvocationInfo().get("run_history"));
+
+ // Test 1 details
+ ITestResult finalTestResult1 = case1.getResult(METHOD_1);
+ assertNotNull(String.format("Expected result for %s", TEST_1), finalTestResult1);
+ assertEquals(
+ String.format("Expected fail for %s", TEST_1),
+ TestStatus.FAIL,
+ finalTestResult1.getResultStatus());
+ assertEquals(finalTestResult1.getStackTrace(), STACK_TRACE);
+
+ // Test 2 details
+ ITestResult finalTestResult2 = case1.getResult(METHOD_2);
+ assertNotNull(String.format("Expected result for %s", TEST_2), finalTestResult2);
+ assertEquals(
+ String.format("Expected pass for %s", TEST_2),
+ TestStatus.PASS,
+ finalTestResult2.getResultStatus());
+ }
}
diff --git a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/suite/PreviousSessionFileCopierTest.java b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/suite/PreviousSessionFileCopierTest.java
index 918acd3..a0c3131 100644
--- a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/suite/PreviousSessionFileCopierTest.java
+++ b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/result/suite/PreviousSessionFileCopierTest.java
@@ -95,4 +95,13 @@
// File are not overriden
assertEquals("CURRENT", FileUtil.readStringFromFile(original));
}
+
+ @Test
+ public void testCopy_fileExcluded() throws Exception {
+ new File(mPreviousDir, "proto").mkdir();
+ mCopier.invocationStarted(mContext);
+ mCopier.invocationEnded(500L);
+ // Nothing was copied
+ assertEquals(0, mCurrentDir.listFiles().length);
+ }
}
diff --git a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/targetprep/BusinessLogicPreparerTest.java b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/targetprep/BusinessLogicPreparerTest.java
index cbffc2f..a32218a 100644
--- a/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/targetprep/BusinessLogicPreparerTest.java
+++ b/common/host-side/tradefed/tests/src/com/android/compatibility/common/tradefed/targetprep/BusinessLogicPreparerTest.java
@@ -169,6 +169,17 @@
testBuildRequestString(14, attributes);
}
+ @Test
+ public void testBuildRequestString_noDeviceInfoJSONFileExists() throws Exception {
+ Map<String, String> attributes = new HashMap<>();
+ // Create a memory device info JSON file for test.
+ File jsonPath = createTestDeviceInfoTextFile("MemoryDeviceInfo");
+ mMockBuildInfo.setFile(DeviceInfoCollector.DEVICE_INFO_DIR, jsonPath, "v1");
+ // Setup BuildInfo attributes.
+ mMockBuildInfo.addBuildAttribute(CompatibilityBuildHelper.SUITE_VERSION, "v1");
+ testBuildRequestString(14, attributes);
+ }
+
private void testBuildRequestString(int expectedParams, Map<String, String> attributes) throws Exception {
for (String key: attributes.keySet()) {
mMockBuildInfo.addBuildAttribute(key, attributes.get(key));
@@ -265,6 +276,12 @@
return mTmpDir;
}
+ private File createTestDeviceInfoTextFile(String DeviceInfoClassName)
+ throws IOException {
+ new File(mTmpDir, DeviceInfoClassName + ".deviceinfo.text");
+ return mTmpDir;
+ }
+
private File createFileFromStr(String configStr) throws IOException {
File file = File.createTempFile("test", "dynamic");
FileOutputStream stream = new FileOutputStream(file);
diff --git a/common/host-side/util/src/com/android/compatibility/common/util/BusinessLogicHostExecutor.java b/common/host-side/util/src/com/android/compatibility/common/util/BusinessLogicHostExecutor.java
index 0976969..9949f07 100644
--- a/common/host-side/util/src/com/android/compatibility/common/util/BusinessLogicHostExecutor.java
+++ b/common/host-side/util/src/com/android/compatibility/common/util/BusinessLogicHostExecutor.java
@@ -70,7 +70,7 @@
*/
@Override
protected String formatExecutionString(String method, String... args) {
- return String.format("%s(%s)", method, String.join(", ", args));
+ return String.format("%s(%s)", method, String.join(", ", formatArgs(args)));
}
/**
diff --git a/common/host-side/util/tests/run_tests.sh b/common/host-side/util/tests/run_tests.sh
new file mode 100755
index 0000000..183cd73
--- /dev/null
+++ b/common/host-side/util/tests/run_tests.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Helper script for running unit tests for compatibility libraries
+
+HARNESS_DIR=$(dirname ${0})/../../../..
+source ${HARNESS_DIR}/test_defs.sh
+
+JARS="
+ compatibility-common-util-hostsidelib\
+ compatibility-common-util-tests\
+ compatibility-host-util\
+ compatibility-host-util-tests\
+ compatibility-mock-tradefed\
+ compatibility-tradefed-tests"
+
+run_tests "com.android.compatibility.common.util.HostUnitTests" "${JARS}" "${@}"
+
diff --git a/common/util/src/com/android/compatibility/common/util/BusinessLogicExecutor.java b/common/util/src/com/android/compatibility/common/util/BusinessLogicExecutor.java
index a53b0c4..34f98d7 100644
--- a/common/util/src/com/android/compatibility/common/util/BusinessLogicExecutor.java
+++ b/common/util/src/com/android/compatibility/common/util/BusinessLogicExecutor.java
@@ -35,6 +35,10 @@
protected static final String STRING_CLASS = "java.lang.String";
protected static final String STRING_ARRAY_CLASS = "[Ljava.lang.String;";
+ /* List of substrings indicating a method arg should be redacted in the logs */
+ private static final String[] REDACTED_VALUES = new String[] {"permission"};
+ private static final String REDACTED_PLACEHOLDER = "[redacted]";
+
/**
* Execute a business logic condition.
* @param method the name of the method to invoke. Must include fully qualified name of the
@@ -92,6 +96,24 @@
*/
protected abstract String formatExecutionString(String method, String... args);
+ /** Substitute sensitive information with REDACTED_PLACEHOLDER if necessary. */
+ protected static String[] formatArgs(String[] args) {
+ List<String> formattedArgs = new ArrayList<>();
+ for (String arg : args) {
+ formattedArgs.add(formatArg(arg));
+ }
+ return formattedArgs.toArray(new String[0]);
+ }
+
+ private static String formatArg(String arg) {
+ for (String str : REDACTED_VALUES) {
+ if (arg.contains(str)) {
+ return REDACTED_PLACEHOLDER;
+ }
+ }
+ return arg;
+ }
+
/**
* Execute a business logic method.
* @param method the name of the method to invoke. Must include fully qualified name of the
diff --git a/common/util/src/com/android/compatibility/common/util/CrashUtils.java b/common/util/src/com/android/compatibility/common/util/CrashUtils.java
index 08375ad..c6433a8 100644
--- a/common/util/src/com/android/compatibility/common/util/CrashUtils.java
+++ b/common/util/src/com/android/compatibility/common/util/CrashUtils.java
@@ -16,8 +16,14 @@
package com.android.compatibility.common.util;
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.math.BigInteger;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -25,7 +31,7 @@
/** Contains helper functions and shared constants for crash parsing. */
public class CrashUtils {
// used to only detect actual addresses instead of nullptr and other unlikely values
- public static final long MIN_CRASH_ADDR = 0x8000;
+ public static final BigInteger MIN_CRASH_ADDR = new BigInteger("8000", 16);
// Matches the end of a crash
public static final Pattern sEndofCrashPattern =
Pattern.compile("DEBUG\\s+?:\\s+?backtrace:");
@@ -37,11 +43,12 @@
public static final String NEW_TEST_ALERT = "New test starting with name: ";
public static final Pattern sNewTestPattern =
Pattern.compile(NEW_TEST_ALERT + "(\\w+?)\\(.*?\\)");
- public static final String SIGNAL = "signal",
- NAME = "name",
- PID = "pid",
- TID = "tid",
- FAULT_ADDRESS = "faultaddress";
+ public static final String SIGNAL = "signal";
+ public static final String NAME = "name";
+ public static final String PROCESS = "process";
+ public static final String PID = "pid";
+ public static final String TID = "tid";
+ public static final String FAULT_ADDRESS = "faultaddress";
// Matches the smallest blob that has the appropriate header and footer
private static final Pattern sCrashBlobPattern =
Pattern.compile("DEBUG\\s+?:( [*]{3})+?.*?DEBUG\\s+?:\\s+?backtrace:", Pattern.DOTALL);
@@ -57,48 +64,90 @@
private static Pattern sAbortMessageCheckPattern =
Pattern.compile("(?i)Abort message.*?CHECK_");
+ public static final String SIGSEGV = "SIGSEGV";
+ public static final String SIGBUS = "SIGBUS";
+ public static final String SIGABRT = "SIGABRT";
+
+ /**
+ * returns the filename of the process.
+ * e.g. "/system/bin/mediaserver" returns "mediaserver"
+ */
+ public static String getProcessFileName(JSONObject crash) throws JSONException {
+ return new File(crash.getString(PROCESS)).getName();
+ }
+
/**
* Determines if the given input has a {@link com.android.compatibility.common.util.Crash} that
* should fail an sts test
*
- * @param processNames list of applicable process names
- * @param checkMinAddr if the minimum fault address should be respected
* @param crashes list of crashes to check
+ * @param config crash detection configuration object
* @return if a crash is serious enough to fail an sts test
*/
- public static boolean detectCrash(
- String[] processNames, boolean checkMinAddr, JSONArray crashes) {
+ public static boolean securityCrashDetected(JSONArray crashes, Config config) {
+ return matchSecurityCrashes(crashes, config).length() > 0;
+ }
+
+ public static BigInteger getBigInteger(JSONObject source, String name) throws JSONException {
+ if (source.isNull(name)) {
+ return null;
+ }
+ String intString = source.getString(name);
+ BigInteger value = null;
+ try {
+ value = new BigInteger(intString, 16);
+ } catch (NumberFormatException e) {}
+ return value;
+ }
+
+ /**
+ * Determines which given inputs have a {@link com.android.compatibility.common.util.Crash} that
+ * should fail an sts test
+ *
+ * @param crashes list of crashes to check
+ * @param config crash detection configuration object
+ * @return the list of crashes serious enough to fail an sts test
+ */
+ public static JSONArray matchSecurityCrashes(JSONArray crashes, Config config) {
+ JSONArray securityCrashes = new JSONArray();
for (int i = 0; i < crashes.length(); i++) {
try {
JSONObject crash = crashes.getJSONObject(i);
- if (!crash.getString(SIGNAL).toLowerCase().matches("sig(segv|bus)")) {
+
+ // match process patterns
+ if (!matchesAny(getProcessFileName(crash), config.processPatterns)) {
continue;
}
- if (checkMinAddr && !crash.isNull(FAULT_ADDRESS)) {
- if (crash.getLong(FAULT_ADDRESS) < MIN_CRASH_ADDR) {
+ // match signal
+ String crashSignal = crash.getString(SIGNAL);
+ if (!config.signals.contains(crashSignal)) {
+ continue;
+ }
+
+ // if check specified, reject crash if address is unlikely to be security-related
+ if (config.checkMinAddress) {
+ BigInteger faultAddress = getBigInteger(crash, FAULT_ADDRESS);
+ if (faultAddress != null
+ && faultAddress.compareTo(config.minCrashAddress) < 0) {
continue;
}
}
+ securityCrashes.put(crash);
+ } catch (JSONException | NullPointerException e) {}
+ }
+ return securityCrashes;
+ }
- boolean foundProcess = false;
- String name = crash.getString(NAME);
- for (String process : processNames) {
- if (name.equals(process)) {
- foundProcess = true;
- break;
- }
- }
-
- if (!foundProcess) {
- continue;
- }
-
- return true; // crash detected
- } catch (JSONException | NullPointerException e) {
+ /**
+ * returns true if the input matches any of the patterns.
+ */
+ private static boolean matchesAny(String input, Collection<Pattern> patterns) {
+ for (Pattern p : patterns) {
+ if (p.matcher(input).matches()) {
+ return true;
}
}
-
return false;
}
@@ -107,21 +156,23 @@
Matcher crashBlobFinder = sCrashBlobPattern.matcher(input);
while (crashBlobFinder.find()) {
String crashStr = crashBlobFinder.group(0);
- int tid = 0, pid = 0;
- Long faultAddress = null;
- String name = null, signal = null;
+ int tid = 0;
+ int pid = 0;
+ BigInteger faultAddress = null;
+ String name = null;
+ String process = null;
+ String signal = null;
Matcher pidtidNameMatcher = sPidtidNamePattern.matcher(crashStr);
if (pidtidNameMatcher.find()) {
try {
pid = Integer.parseInt(pidtidNameMatcher.group(1));
- } catch (NumberFormatException e) {
- }
+ } catch (NumberFormatException e) {}
try {
tid = Integer.parseInt(pidtidNameMatcher.group(2));
- } catch (NumberFormatException e) {
- }
+ } catch (NumberFormatException e) {}
name = pidtidNameMatcher.group(3).trim();
+ process = pidtidNameMatcher.group(4).trim();
}
Matcher faultLineMatcher = sFaultLinePattern.matcher(crashStr);
@@ -130,9 +181,8 @@
String faultAddrMatch = faultLineMatcher.group(2);
if (faultAddrMatch != null) {
try {
- faultAddress = Long.parseLong(faultAddrMatch, 16);
- } catch (NumberFormatException e) {
- }
+ faultAddress = new BigInteger(faultAddrMatch, 16);
+ } catch (NumberFormatException e) {}
}
}
if (!sAbortMessageCheckPattern.matcher(crashStr).find()) {
@@ -141,14 +191,63 @@
crash.put(PID, pid);
crash.put(TID, tid);
crash.put(NAME, name);
- crash.put(FAULT_ADDRESS, faultAddress);
+ crash.put(PROCESS, process);
+ crash.put(FAULT_ADDRESS,
+ faultAddress == null ? null : faultAddress.toString(16));
crash.put(SIGNAL, signal);
crashes.put(crash);
- } catch (JSONException e) {
-
- }
+ } catch (JSONException e) {}
}
}
return crashes;
}
+
+ public static class Config {
+ private boolean checkMinAddress = true;
+ private BigInteger minCrashAddress = MIN_CRASH_ADDR;
+ private List<String> signals = Arrays.asList(SIGSEGV, SIGBUS);
+ private List<Pattern> processPatterns = Collections.emptyList();
+
+ public Config setMinAddress(BigInteger minCrashAddress) {
+ this.minCrashAddress = minCrashAddress;
+ return this;
+ }
+
+ public Config checkMinAddress(boolean checkMinAddress) {
+ this.checkMinAddress = checkMinAddress;
+ return this;
+ }
+
+ public Config setSignals(String... signals) {
+ this.signals = Arrays.asList(signals);
+ return this;
+ }
+
+ public Config appendSignals(String... signals) {
+ Collections.addAll(this.signals, signals);
+ return this;
+ }
+
+ public Config setProcessPatterns(String... processPatternStrings) {
+ Pattern[] processPatterns = new Pattern[processPatternStrings.length];
+ for (int i = 0; i < processPatternStrings.length; i++) {
+ processPatterns[i] = Pattern.compile(processPatternStrings[i]);
+ }
+ return setProcessPatterns(processPatterns);
+ }
+
+ public Config setProcessPatterns(Pattern... processPatterns) {
+ this.processPatterns = Arrays.asList(processPatterns);
+ return this;
+ }
+
+ public List<Pattern> getProcessPatterns() {
+ return Collections.unmodifiableList(processPatterns);
+ }
+
+ public Config appendProcessPatterns(Pattern... processPatterns) {
+ Collections.addAll(this.processPatterns, processPatterns);
+ return this;
+ }
+ }
}
diff --git a/common/util/src/com/android/compatibility/common/util/ITestResult.java b/common/util/src/com/android/compatibility/common/util/ITestResult.java
index d5c95e8..33340e6 100644
--- a/common/util/src/com/android/compatibility/common/util/ITestResult.java
+++ b/common/util/src/com/android/compatibility/common/util/ITestResult.java
@@ -15,6 +15,8 @@
*/
package com.android.compatibility.common.util;
+import java.util.List;
+
/**
* Represents a single test result.
*/
@@ -150,4 +152,22 @@
* Clear the existing result and default to 'failed'
*/
void removeResult();
+
+ /**
+ * This method is to record per-case history for CTS Verifier. If this field is used for large
+ * test suites like CTS, it may cause performance issues in APFE. Thus please do not use this
+ * field in other test suites.
+ *
+ * @return The test result histories
+ */
+ List<TestResultHistory> getTestResultHistories();
+
+ /**
+ * Set test result histories of test item. This method is for per-case history in CTS Verifier.
+ * If this field is used for large test suites like CTS, it may cause performance issues in
+ * APFE. Thus please do not use this field in other test suites.
+ *
+ * @param resultHistories The test result histories.
+ */
+ void setTestResultHistories(List<TestResultHistory> resultHistories);
}
diff --git a/common/util/src/com/android/compatibility/common/util/InvocationResult.java b/common/util/src/com/android/compatibility/common/util/InvocationResult.java
index 45780a9..969e188 100644
--- a/common/util/src/com/android/compatibility/common/util/InvocationResult.java
+++ b/common/util/src/com/android/compatibility/common/util/InvocationResult.java
@@ -17,6 +17,7 @@
import java.io.File;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -30,6 +31,13 @@
*/
public class InvocationResult implements IInvocationResult {
+ /** Helper object for JSON conversion. */
+ public static final class RunHistory {
+ public long startTime;
+ public long endTime;
+ }
+
+ private Collection<RunHistory> mRunHistories = new ArrayList<>();
private long mTimestamp;
private Map<String, IModuleResult> mModuleResults = new LinkedHashMap<>();
private Map<String, String> mInvocationInfo = new HashMap<>();
@@ -40,6 +48,11 @@
private RetryChecksumStatus mRetryChecksumStatus = RetryChecksumStatus.NotRetry;
private File mRetryDirectory = null;
+ /** @return a collection of the run history of previous runs. */
+ public Collection<RunHistory> getRunHistories() {
+ return mRunHistories;
+ }
+
/**
* {@inheritDoc}
*/
diff --git a/common/util/src/com/android/compatibility/common/util/ResultHandler.java b/common/util/src/com/android/compatibility/common/util/ResultHandler.java
index 1be3624..edf457c 100644
--- a/common/util/src/com/android/compatibility/common/util/ResultHandler.java
+++ b/common/util/src/com/android/compatibility/common/util/ResultHandler.java
@@ -37,6 +37,7 @@
import java.nio.file.Path;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
@@ -104,6 +105,9 @@
private static final String RESULT_ATTR = "result";
private static final String RESULT_TAG = "Result";
private static final String RUNTIME_ATTR = "runtime";
+ private static final String RUN_HISTORY_ATTR = "run_history";
+ private static final String RUN_HISTORY_TAG = "RunHistory";
+ private static final String RUN_TAG = "Run";
private static final String SCREENSHOT_TAG = "Screenshot";
private static final String SKIPPED_ATTR = "skipped";
private static final String STACK_TAG = "StackTrace";
@@ -198,6 +202,10 @@
invocation.addInvocationInfo(BUILD_ID, parser.getAttributeValue(NS, BUILD_ID));
invocation.addInvocationInfo(BUILD_PRODUCT, parser.getAttributeValue(NS,
BUILD_PRODUCT));
+ String runHistoryValue = parser.getAttributeValue(NS, RUN_HISTORY_ATTR);
+ if (runHistoryValue != null) {
+ invocation.addInvocationInfo(RUN_HISTORY_ATTR, runHistoryValue);
+ }
// The build fingerprint needs to reflect the true fingerprint of the device under test,
// ignoring potential overrides made by test suites (namely STS) for APFE build
@@ -212,7 +220,19 @@
// --skip-device-info flag
parser.nextTag();
parser.require(XmlPullParser.END_TAG, NS, BUILD_TAG);
+
+ // Parse RunHistory tag.
parser.nextTag();
+ boolean hasRunHistoryTag = true;
+ try {
+ parser.require(parser.START_TAG, NS, RUN_HISTORY_TAG);
+ } catch (XmlPullParserException e) {
+ hasRunHistoryTag = false;
+ }
+ if (hasRunHistoryTag) {
+ parseRunHistory(parser);
+ }
+
parser.require(XmlPullParser.START_TAG, NS, SUMMARY_TAG);
parser.nextTag();
parser.require(XmlPullParser.END_TAG, NS, SUMMARY_TAG);
@@ -269,6 +289,10 @@
// Ignore the new format in the old parser.
parser.nextText();
parser.require(XmlPullParser.END_TAG, NS, METRIC_TAG);
+ } else if (RUN_HISTORY_TAG.equals(parser.getName())) {
+ // Ignore the test result history since it only exists in
+ // CTS Verifier, which will not use parsing feature.
+ skipCurrentTag(parser);
} else {
parser.nextTag();
}
@@ -308,6 +332,34 @@
}
}
+ /** Parse and replay all run history information. */
+ private static void parseRunHistory(XmlPullParser parser)
+ throws IOException, XmlPullParserException {
+ while (parser.nextTag() == XmlPullParser.START_TAG) {
+ parser.require(XmlPullParser.START_TAG, NS, RUN_TAG);
+ parser.nextTag();
+ parser.require(XmlPullParser.END_TAG, NS, RUN_TAG);
+ }
+ parser.require(XmlPullParser.END_TAG, NS, RUN_HISTORY_TAG);
+ parser.nextTag();
+ }
+
+ /** Skip the current XML tags. */
+ private static void skipCurrentTag(XmlPullParser parser)
+ throws XmlPullParserException, IOException {
+ int depth = 1;
+ while (depth != 0) {
+ switch (parser.next()) {
+ case XmlPullParser.END_TAG:
+ depth--;
+ break;
+ case XmlPullParser.START_TAG:
+ depth++;
+ break;
+ }
+ }
+ }
+
/**
* @param result
* @param resultDir
@@ -392,6 +444,21 @@
}
serializer.endTag(NS, BUILD_TAG);
+ // Run history - this contains a list of start and end times of previous runs. More
+ // information may be added in the future.
+ Collection<InvocationResult.RunHistory> runHistories =
+ ((InvocationResult) result).getRunHistories();
+ if (!runHistories.isEmpty()) {
+ serializer.startTag(NS, RUN_HISTORY_TAG);
+ for (InvocationResult.RunHistory runHistory : runHistories) {
+ serializer.startTag(NS, RUN_TAG);
+ serializer.attribute(NS, START_TIME_ATTR, String.valueOf(runHistory.startTime));
+ serializer.attribute(NS, END_TIME_ATTR, String.valueOf(runHistory.endTime));
+ serializer.endTag(NS, RUN_TAG);
+ }
+ serializer.endTag(NS, RUN_HISTORY_TAG);
+ }
+
// Summary
serializer.startTag(NS, SUMMARY_TAG);
serializer.attribute(NS, PASS_ATTR, Integer.toString(passed));
@@ -459,6 +526,15 @@
if (report != null) {
ReportLog.serialize(serializer, report);
}
+
+ // Test result history contains a list of execution time for each test item.
+ List<TestResultHistory> testResultHistories = r.getTestResultHistories();
+ if (testResultHistories != null) {
+ for (TestResultHistory resultHistory : testResultHistories) {
+ TestResultHistory.serialize(serializer, resultHistory, r.getName());
+ }
+ }
+
serializer.endTag(NS, TEST_TAG);
}
serializer.endTag(NS, CASE_TAG);
diff --git a/common/util/src/com/android/compatibility/common/util/ResultUnit.java b/common/util/src/com/android/compatibility/common/util/ResultUnit.java
index 31aecbb..131ba8f 100644
--- a/common/util/src/com/android/compatibility/common/util/ResultUnit.java
+++ b/common/util/src/com/android/compatibility/common/util/ResultUnit.java
@@ -39,7 +39,9 @@
/** unit for benchmarking with generic score. */
SCORE,
/** radian */
- RADIAN;
+ RADIAN,
+ /** Audio or Video frames count, dropped, repeated, etc... */
+ FRAMES;
/**
* @return a string to be used in the report.
diff --git a/common/util/src/com/android/compatibility/common/util/TestResult.java b/common/util/src/com/android/compatibility/common/util/TestResult.java
index 18a8b4c..d2a9ca9 100644
--- a/common/util/src/com/android/compatibility/common/util/TestResult.java
+++ b/common/util/src/com/android/compatibility/common/util/TestResult.java
@@ -15,6 +15,8 @@
*/
package com.android.compatibility.common.util;
+import java.util.List;
+
/**
* Represents a single test result.
*/
@@ -31,6 +33,7 @@
private String mScreenshot;
private boolean mIsRetry;
private boolean mSkipped;
+ private List<TestResultHistory> mTestResultHistories;
/**
* Create a {@link TestResult} for the given test name.
@@ -233,6 +236,7 @@
mScreenshot = null;
mIsRetry = false;
mSkipped = false;
+ mTestResultHistories = null;
}
/**
@@ -280,4 +284,19 @@
}
}
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<TestResultHistory> getTestResultHistories() {
+ return mTestResultHistories;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setTestResultHistories(List<TestResultHistory> resultHistories) {
+ mTestResultHistories = resultHistories;
+ }
}
diff --git a/common/util/src/com/android/compatibility/common/util/TestResultHistory.java b/common/util/src/com/android/compatibility/common/util/TestResultHistory.java
new file mode 100644
index 0000000..12f1da6
--- /dev/null
+++ b/common/util/src/com/android/compatibility/common/util/TestResultHistory.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.compatibility.common.util;
+
+import org.xmlpull.v1.XmlPullParserException;
+import org.xmlpull.v1.XmlPullParserFactory;
+import org.xmlpull.v1.XmlSerializer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Utility class to add test case result history to the report. This class records per-case history
+ * for CTS Verifier. If this field is used for large test suites like CTS, it may cause performance
+ * issues in APFE. Thus please do not use this class in other test suites.
+ */
+public class TestResultHistory implements Serializable {
+
+ private static final long serialVersionUID = 10L;
+
+ private static final String ENCODING = "UTF-8";
+ private static final String TYPE = "org.kxml2.io.KXmlParser,org.kxml2.io.KXmlSerializer";
+
+ // XML constants
+ private static final String SUB_TEST_ATTR = "subtest";
+ private static final String RUN_HISTORY_TAG = "RunHistory";
+ private static final String RUN_TAG = "Run";
+ private static final String START_TIME_ATTR = "start";
+ private static final String END_TIME_ATTR = "end";
+
+ private final String mTestName;
+ private final Set<Map.Entry> mDurations;
+
+ /**
+ * Constructor of test result history.
+ *
+ * @param testName a string of test name.
+ * @param durations a set of execution time.
+ */
+ public TestResultHistory(String testName, Set<Map.Entry> durations) {
+ this.mTestName = testName;
+ this.mDurations = durations;
+ }
+
+ /** Get test name */
+ public String getTestName() {
+ return mTestName;
+ }
+
+ /** Get a set of duration with key as start time and value as end time. */
+ public Set<Map.Entry> getDurations() {
+ return mDurations;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ TestResultHistory that = (TestResultHistory) o;
+ return Objects.equals(mTestName, that.mTestName)
+ && Objects.equals(mDurations, that.mDurations);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public int hashCode() {
+ return Objects.hash(mTestName, mDurations);
+ }
+
+ /**
+ * Serializes a given {@link TestResultHistory} to XML.
+ *
+ * @param serializer given serializer.
+ * @param resultHistory test result history with test name and execution time.
+ * @param testName top-level test name.
+ * @throws IOException
+ */
+ public static void serialize(
+ XmlSerializer serializer, TestResultHistory resultHistory, String testName)
+ throws IOException {
+ if (resultHistory == null) {
+ throw new IllegalArgumentException("Test result history was null");
+ }
+
+ serializer.startTag(null, RUN_HISTORY_TAG);
+ // Only show sub-test names in test attribute in run history node.
+ String name = resultHistory.getTestName().replaceFirst(testName + ":", "");
+ if (!name.isEmpty() && !name.equalsIgnoreCase(testName)) {
+ serializer.attribute(null, SUB_TEST_ATTR, name);
+ }
+
+ for (Map.Entry<Long, Long> duration : resultHistory.getDurations()) {
+ serializer.startTag(null, RUN_TAG);
+ serializer.attribute(null, START_TIME_ATTR, String.valueOf(duration.getKey()));
+ serializer.attribute(null, END_TIME_ATTR, String.valueOf(duration.getValue()));
+ serializer.endTag(null, RUN_TAG);
+ }
+ serializer.endTag(null, RUN_HISTORY_TAG);
+ }
+
+ /**
+ * Serializes a given {@link TestResultHistory} to a String.
+ *
+ * @param resultHistory test result history with test name and execution time.
+ * @param testName top-level test name.
+ * @throws XmlPullParserException
+ * @throws IOException
+ * @throws IllegalStateException
+ * @throws IllegalArgumentException
+ */
+ public static String serialize(TestResultHistory resultHistory, String testName)
+ throws XmlPullParserException, IllegalArgumentException, IllegalStateException,
+ IOException {
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ XmlSerializer serializer = XmlPullParserFactory.newInstance(TYPE, null).newSerializer();
+ serializer.setOutput(byteArrayOutputStream, ENCODING);
+ serializer.startDocument(ENCODING, true);
+ serializer.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
+ serialize(serializer, resultHistory, testName);
+ serializer.endDocument();
+ return byteArrayOutputStream.toString(ENCODING);
+ }
+}
diff --git a/common/util/src/com/android/compatibility/common/util/VersionCodes.java b/common/util/src/com/android/compatibility/common/util/VersionCodes.java
index d67037d..ddc8249 100644
--- a/common/util/src/com/android/compatibility/common/util/VersionCodes.java
+++ b/common/util/src/com/android/compatibility/common/util/VersionCodes.java
@@ -46,5 +46,6 @@
public static final int O = 26;
public static final int O_MR1 = 27;
public static final int P = 28;
- public static final int Q = CUR_DEVELOPMENT;
+ public static final int Q = 29;
+ public static final int R = CUR_DEVELOPMENT;
}
diff --git a/common/util/tests/assets/logcat.txt b/common/util/tests/assets/logcat.txt
index ad778c7..b9d10d0 100644
--- a/common/util/tests/assets/logcat.txt
+++ b/common/util/tests/assets/logcat.txt
@@ -271,4 +271,66 @@
11-25 19:47:35.597 940 940 F DEBUG : #24 pc 0003fa3b /system/lib/libc.so (_ZL15__pthread_startPv+30)
11-25 19:47:35.597 940 940 F DEBUG : #25 pc 0001a085 /system/lib/libc.so (__start_thread+6)
11-25 19:47:35.837 940 940 F DEBUG :
-11-25 19:47:35.837 940 940 F DEBUG : Tombstone written to: /data/tombstones/tombstone_01
\ No newline at end of file
+11-25 19:47:35.837 940 940 F DEBUG : Tombstone written to: /data/tombstones/tombstone_01
+--------- beginning of crash
+09-03 17:48:05.627 11071 11189 F libc : Fatal signal 11 (SIGSEGV), code 1, fault addr 0xe9380000 in tid 11189 (synthetic_thread)
+09-03 17:48:05.707 359 359 W : debuggerd: handling request: pid=11071 uid=1041 gid=1005 tid=11189
+09-03 17:48:05.796 7072 7072 F DEBUG : *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
+09-03 17:48:05.796 7072 7072 F DEBUG : Build fingerprint: 'google/angler/angler:7.1.1/N4F26T/3687331:userdebug/dev-keys'
+09-03 17:48:05.796 7072 7072 F DEBUG : Revision: '0'
+09-03 17:48:05.796 7072 7072 F DEBUG : ABI: 'arm'
+09-03 17:48:05.796 7072 7072 F DEBUG : pid: 11071, tid: 11189, name: synthetic_thread >>> synthetic_process_0 <<<
+09-03 17:48:05.797 7072 7072 F DEBUG : signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0xe9380000
+09-03 17:48:05.797 7072 7072 F DEBUG : r0 e9e7a240 r1 e9380000 r2 00000170 r3 00000000
+09-03 17:48:05.797 7072 7072 F DEBUG : r4 00000002 r5 00000000 r6 ec1e1f25 r7 eb6f8000
+09-03 17:48:05.797 7072 7072 F DEBUG : r8 00000000 r9 eb105204 sl 00000000 fp 000003c0
+09-03 17:48:05.797 7072 7072 F DEBUG : ip ebd3df18 sp eaf80688 lr ec1e1f41 pc ebd38dd6 cpsr 20000030
+09-03 17:48:05.805 7072 7072 F DEBUG :
+09-03 17:48:05.805 7072 7072 F DEBUG : backtrace:
+09-03 17:48:05.806 7072 7072 F DEBUG : #00 pc 00002dd6 /system/lib/libaudioutils.so (memcpy_to_float_from_i16+5)
+09-03 17:48:05.806 7072 7072 F DEBUG : #01 pc 00040f3d /system/lib/libaudioflinger.so
+09-03 17:48:05.806 7072 7072 F DEBUG : #02 pc 00040799 /system/lib/libaudioflinger.so
+09-03 17:48:05.806 7072 7072 F DEBUG : #03 pc 00011178 /system/lib/libaudioflinger.so
+09-03 17:48:05.806 7072 7072 F DEBUG : #04 pc 0003180b /system/lib/libaudioflinger.so
+09-03 17:48:05.806 7072 7072 F DEBUG : #05 pc 0002fe57 /system/lib/libaudioflinger.so
+09-03 17:48:05.806 7072 7072 F DEBUG : #06 pc 0000e345 /system/lib/libutils.so (_ZN7android6Thread11_threadLoopEPv+140)
+09-03 17:48:05.806 7072 7072 F DEBUG : #07 pc 000470b3 /system/lib/libc.so (_ZL15__pthread_startPv+22)
+09-03 17:48:05.806 7072 7072 F DEBUG : #08 pc 00019e3d /system/lib/libc.so (__start_thread+6)
+09-03 17:48:05.967 11272 11568 W NativeCrashListener: Couldn't find ProcessRecord for pid 11071
+09-03 17:48:05.969 359 359 W : debuggerd: resuming target 11071
+09-03 17:48:05.981 11272 11307 I BootReceiver: Copying /data/tombstones/tombstone_01 to DropBox (SYSTEM_TOMBSTONE)
+09-03 17:48:06.067 394 394 I ServiceManager: service 'media.sound_trigger_hw' died
+06-15 19:57:33.607 12736 12761 D PermissionCache: checking android.permission.MODIFY_AUDIO_SETTINGS for uid=10197 => granted (698 us)
+--------- beginning of crash
+06-15 19:57:33.607 12736 12761 F libc : Fatal signal 11 (SIGSEGV), code 1, fault addr 0x0 in tid 12761 (synthetic_thread)
+06-15 19:57:33.608 379 379 W : debuggerd: handling request: pid=12736 uid=1041 gid=1005 tid=12761
+06-15 19:57:33.670 26192 26192 F DEBUG : *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** ***
+06-15 19:57:33.670 26192 26192 F DEBUG : Build fingerprint: 'google/bullhead/bullhead:7.1.2/N2G48C/4104010:userdebug/dev-keys'
+06-15 19:57:33.670 26192 26192 F DEBUG : Revision: 'rev_1.0'
+06-15 19:57:33.670 26192 26192 F DEBUG : ABI: 'arm'
+06-15 19:57:33.670 26192 26192 F DEBUG : pid: 12736, tid: 12761, name: synthetic_thread >>> synthetic_process_1 <<<
+06-15 19:57:33.670 26192 26192 F DEBUG : signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0x0
+06-15 19:57:33.670 26192 26192 F DEBUG : r0 00000000 r1 00000000 r2 0000005f r3 00000000
+06-15 19:57:33.670 26192 26192 F DEBUG : r4 ffffffff r5 00000000 r6 f14f9000 r7 00000001
+06-15 19:57:33.670 26192 26192 F DEBUG : r8 00000004 r9 f3353114 sl f3313900 fp 00000000
+06-15 19:57:33.670 26192 26192 F DEBUG : ip f3bd4d88 sp f127d9c8 lr f3b9cbc5 pc f3b65af4 cpsr 60000030
+06-15 19:57:33.676 26192 26192 F DEBUG :
+06-15 19:57:33.676 26192 26192 F DEBUG : backtrace:
+06-15 19:57:33.677 26192 26192 F DEBUG : #00 pc 00018af4 /system/lib/libc.so (strlen+71)
+06-15 19:57:33.677 26192 26192 F DEBUG : #01 pc 0004fbc1 /system/lib/libc.so (__strlen_chk+4)
+06-15 19:57:33.677 26192 26192 F DEBUG : #02 pc 0000c599 /system/lib/libutils.so (_ZN7android7String8C2EPKc+12)
+06-15 19:57:33.677 26192 26192 F DEBUG : #03 pc 0002fdbf /system/lib/libaudiopolicymanagerdefault.so (_ZNK7android18HwModuleCollection19getDeviceDescriptorEjPKcS2_b+458)
+06-15 19:57:33.677 26192 26192 F DEBUG : #04 pc 0001de47 /system/lib/libaudiopolicymanagerdefault.so (_ZN7android18AudioPolicyManager27setDeviceConnectionStateIntEj24audio_policy_dev_state_tPKcS3_+178)
+06-15 19:57:33.677 26192 26192 F DEBUG : #05 pc 0000a009 /system/lib/libaudiopolicyservice.so
+06-15 19:57:33.677 26192 26192 F DEBUG : #06 pc 000a01a5 /system/lib/libmedia.so (_ZN7android20BnAudioPolicyService10onTransactEjRKNS_6ParcelEPS1_j+1256)
+06-15 19:57:33.677 26192 26192 F DEBUG : #07 pc 000359c3 /system/lib/libbinder.so (_ZN7android7BBinder8transactEjRKNS_6ParcelEPS1_j+70)
+06-15 19:57:33.677 26192 26192 F DEBUG : #08 pc 0003d1bb /system/lib/libbinder.so (_ZN7android14IPCThreadState14executeCommandEi+702)
+06-15 19:57:33.677 26192 26192 F DEBUG : #09 pc 0003ce07 /system/lib/libbinder.so (_ZN7android14IPCThreadState20getAndExecuteCommandEv+114)
+06-15 19:57:33.677 26192 26192 F DEBUG : #10 pc 0003d31b /system/lib/libbinder.so (_ZN7android14IPCThreadState14joinThreadPoolEb+46)
+06-15 19:57:33.678 26192 26192 F DEBUG : #11 pc 0004f8c5 /system/lib/libbinder.so
+06-15 19:57:33.678 26192 26192 F DEBUG : #12 pc 0000e345 /system/lib/libutils.so (_ZN7android6Thread11_threadLoopEPv+140)
+06-15 19:57:33.678 26192 26192 F DEBUG : #13 pc 000470b3 /system/lib/libc.so (_ZL15__pthread_startPv+22)
+06-15 19:57:33.678 26192 26192 F DEBUG : #14 pc 00019e3d /system/lib/libc.so (__start_thread+6)
+06-15 19:57:33.839 934 2991 W NativeCrashListener: Couldn't find ProcessRecord for pid 12736
+06-15 19:57:33.846 934 952 I BootReceiver: Copying /data/tombstones/tombstone_01 to DropBox (SYSTEM_TOMBSTONE)
+
diff --git a/common/util/tests/run_tests.sh b/common/util/tests/run_tests.sh
new file mode 100755
index 0000000..bed2c19
--- /dev/null
+++ b/common/util/tests/run_tests.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HARNESS_DIR=$(dirname ${0})/../../..
+source ${HARNESS_DIR}/test_defs.sh
+
+JARS="
+ compatibility-common-util-hostsidelib\
+ compatibility-common-util-tests\
+ compatibility-host-util\
+ compatibility-host-util-tests\
+ compatibility-mock-tradefed\
+ compatibility-tradefed-tests"
+
+run_tests "com.android.compatibility.common.util.UnitTests" "${JARS}" "${@}"
+
diff --git a/common/util/tests/src/com/android/compatibility/common/util/CrashUtilsTest.java b/common/util/tests/src/com/android/compatibility/common/util/CrashUtilsTest.java
index 94f472e..071733f 100644
--- a/common/util/tests/src/com/android/compatibility/common/util/CrashUtilsTest.java
+++ b/common/util/tests/src/com/android/compatibility/common/util/CrashUtilsTest.java
@@ -27,6 +27,7 @@
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import java.util.regex.Pattern;
/** Unit tests for {@link CrashUtils}. */
@RunWith(JUnit4.class)
@@ -52,67 +53,99 @@
@Test
public void testGetAllCrashes() throws Exception {
JSONArray expectedResults = new JSONArray();
- expectedResults.put(createCrashJson(11071, 11189, "AudioOut_D", 3912761344L, "SIGSEGV"));
- expectedResults.put(createCrashJson(12736, 12761, "Binder:12736_2", 0L, "SIGSEGV"));
- expectedResults.put(createCrashJson(26201, 26227, "Binder:26201_3", 0L, "SIGSEGV"));
- expectedResults.put(createCrashJson(26246, 26282, "Binder:26246_5", 0L, "SIGSEGV"));
- expectedResults.put(createCrashJson(245, 245, "installd", null, "SIGABRT"));
- expectedResults.put(createCrashJson(6371, 8072, "media.codec", 3976200192L, "SIGSEGV"));
- expectedResults.put(createCrashJson(8373, 8414, "loo", null, "SIGABRT"));
+ expectedResults.put(createCrashJson(
+ 11071, 11189, "AudioOut_D", "/system/bin/audioserver", "e9380000", "SIGSEGV"));
+ expectedResults.put(createCrashJson(
+ 12736, 12761, "Binder:12736_2", "/system/bin/audioserver", "0", "SIGSEGV"));
+ expectedResults.put(createCrashJson(
+ 26201, 26227, "Binder:26201_3", "/system/bin/audioserver", "0", "SIGSEGV"));
+ expectedResults.put(createCrashJson(
+ 26246, 26282, "Binder:26246_5", "/system/bin/audioserver", "0", "SIGSEGV"));
+ expectedResults.put(createCrashJson(
+ 245, 245, "installd", "/system/bin/installd", null, "SIGABRT"));
+ expectedResults.put(createCrashJson(
+ 6371, 8072, "media.codec", "omx@1.0-service", "ed000000", "SIGSEGV"));
+ expectedResults.put(createCrashJson(
+ 8373, 8414, "loo", "com.android.bluetooth", null, "SIGABRT"));
+ expectedResults.put(createCrashJson(
+ 11071, 11189, "synthetic_thread", "synthetic_process_0", "e9380000", "SIGSEGV"));
+ expectedResults.put(createCrashJson(
+ 12736, 12761, "synthetic_thread", "synthetic_process_1", "0", "SIGSEGV"));
Assert.assertEquals(mCrashes.toString(), expectedResults.toString());
}
public JSONObject createCrashJson(
- int pid, int tid, String name, Long faultaddress, String signal) {
+ int pid, int tid, String name, String process, String faultaddress, String signal) {
JSONObject json = new JSONObject();
try {
json.put(CrashUtils.PID, pid);
json.put(CrashUtils.TID, tid);
json.put(CrashUtils.NAME, name);
+ json.put(CrashUtils.PROCESS, process);
json.put(CrashUtils.FAULT_ADDRESS, faultaddress);
json.put(CrashUtils.SIGNAL, signal);
- } catch (JSONException e) {
-
- }
+ } catch (JSONException e) {}
return json;
}
@Test
public void testValidCrash() throws Exception {
- Assert.assertTrue(CrashUtils.detectCrash(new String[] {"AudioOut_D"}, true, mCrashes));
+ Assert.assertTrue(CrashUtils.securityCrashDetected(mCrashes, new CrashUtils.Config()
+ .checkMinAddress(true)
+ .setProcessPatterns(Pattern.compile("synthetic_process_0"))));
}
@Test
public void testMissingName() throws Exception {
- Assert.assertFalse(CrashUtils.detectCrash(new String[] {""}, true, mCrashes));
+ Assert.assertFalse(CrashUtils.securityCrashDetected(mCrashes, new CrashUtils.Config()
+ .checkMinAddress(true)
+ .setProcessPatterns(Pattern.compile(""))));
}
@Test
public void testSIGABRT() throws Exception {
- Assert.assertFalse(CrashUtils.detectCrash(new String[] {"installd"}, true, mCrashes));
+ Assert.assertFalse(CrashUtils.securityCrashDetected(mCrashes, new CrashUtils.Config()
+ .checkMinAddress(true)
+ .setProcessPatterns(Pattern.compile("installd"))));
}
@Test
public void testFaultAddressBelowMin() throws Exception {
- Assert.assertFalse(CrashUtils.detectCrash(new String[] {"Binder:12736_2"}, true, mCrashes));
+ Assert.assertFalse(CrashUtils.securityCrashDetected(mCrashes, new CrashUtils.Config()
+ .checkMinAddress(true)
+ .setProcessPatterns(Pattern.compile("synthetic_process_1"))));
}
@Test
public void testIgnoreMinAddressCheck() throws Exception {
- Assert.assertTrue(CrashUtils.detectCrash(new String[] {"Binder:12736_2"}, false, mCrashes));
+ Assert.assertTrue(CrashUtils.securityCrashDetected(mCrashes, new CrashUtils.Config()
+ .checkMinAddress(false)
+ .setProcessPatterns(Pattern.compile("synthetic_process_1"))));
+ }
+
+ @Test
+ public void testBadAbortMessage() throws Exception {
+ Assert.assertFalse(CrashUtils.securityCrashDetected(mCrashes, new CrashUtils.Config()
+ .checkMinAddress(true)
+ .setProcessPatterns(Pattern.compile("generic"))));
}
@Test
public void testGoodAndBadCrashes() throws Exception {
- Assert.assertTrue(
- CrashUtils.detectCrash(new String[] {"AudioOut_D", "generic"}, true, mCrashes));
+ Assert.assertTrue(CrashUtils.securityCrashDetected(mCrashes, new CrashUtils.Config()
+ .checkMinAddress(true)
+ .setProcessPatterns(
+ Pattern.compile("synthetic_process_0"),
+ Pattern.compile("generic"))));
}
@Test
public void testNullFaultAddress() throws Exception {
JSONArray crashes = new JSONArray();
- crashes.put(createCrashJson(8373, 8414, "loo", null, "SIGSEGV"));
- Assert.assertTrue(CrashUtils.detectCrash(new String[] {"loo"}, true, crashes));
+ crashes.put(createCrashJson(8373, 8414, "loo", "com.android.bluetooth", null, "SIGSEGV"));
+ Assert.assertTrue(CrashUtils.securityCrashDetected(crashes, new CrashUtils.Config()
+ .checkMinAddress(true)
+ .setProcessPatterns(Pattern.compile("com\\.android\\.bluetooth"))));
}
}
diff --git a/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java b/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java
index 50a00f0..be22964 100644
--- a/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java
+++ b/common/util/tests/src/com/android/compatibility/common/util/ResultHandlerTest.java
@@ -18,18 +18,32 @@
import com.android.tradefed.util.FileUtil;
import com.android.tradefed.util.AbiUtils;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
import junit.framework.TestCase;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.StringReader;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.AbstractMap;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+
/**
* Unit tests for {@link ResultHandler}
*/
@@ -59,10 +73,14 @@
private static final String BUILD_FINGERPRINT_UNALTERED = "build_fingerprint_unaltered";
private static final String BUILD_ID = "build_id";
private static final String BUILD_PRODUCT = "build_product";
+ private static final String RUN_HISTORY = "run_history";
private static final String EXAMPLE_BUILD_ID = "XYZ";
private static final String EXAMPLE_BUILD_PRODUCT = "wolverine";
private static final String EXAMPLE_BUILD_FINGERPRINT = "example_build_fingerprint";
private static final String EXAMPLE_BUILD_FINGERPRINT_UNALTERED = "example_build_fingerprint_unaltered";
+ private static final String EXAMPLE_RUN_HISTORY =
+ "[{\"startTime\":10000000000000,\"endTime\":10000000000001},"
+ + "{\"startTime\":10000000000002,\"endTime\":10000000000003}]";
private static final String DEVICE_A = "device123";
private static final String DEVICE_B = "device456";
@@ -87,6 +105,8 @@
private static final long END_MS = 1431673199000L;
private static final String START_DISPLAY = "Fri Aug 20 15:13:03 PDT 2010";
private static final String END_DISPLAY = "Fri Aug 20 15:13:04 PDT 2010";
+ private static final long TEST_START_MS = 1000000000011L;
+ private static final long TEST_END_MS = 1000000000012L;
private static final String REFERENCE_URL="http://android.com";
private static final String LOG_URL ="file:///path/to/logs";
@@ -176,6 +196,18 @@
result.addInvocationInfo(BUILD_FINGERPRINT, EXAMPLE_BUILD_FINGERPRINT);
result.addInvocationInfo(BUILD_ID, EXAMPLE_BUILD_ID);
result.addInvocationInfo(BUILD_PRODUCT, EXAMPLE_BUILD_PRODUCT);
+ result.addInvocationInfo(RUN_HISTORY, EXAMPLE_RUN_HISTORY);
+ Collection<InvocationResult.RunHistory> runHistories =
+ ((InvocationResult) result).getRunHistories();
+ InvocationResult.RunHistory runHistory1 = new InvocationResult.RunHistory();
+ runHistory1.startTime = 10000000000000L;
+ runHistory1.endTime = 10000000000001L;
+ runHistories.add(runHistory1);
+ InvocationResult.RunHistory runHistory2 = new InvocationResult.RunHistory();
+ runHistory2.startTime = 10000000000002L;
+ runHistory2.endTime = 10000000000003L;
+ runHistories.add(runHistory2);
+
// Module A: test1 passes, test2 not executed
IModuleResult moduleA = result.getOrCreateModule(ID_A);
moduleA.setDone(false);
@@ -208,12 +240,116 @@
moduleBTest5.skipped();
// Serialize to file
- ResultHandler.writeResults(SUITE_NAME, SUITE_VERSION, SUITE_PLAN, SUITE_BUILD,
- result, resultDir, START_MS, END_MS, REFERENCE_URL, LOG_URL,
- COMMAND_LINE_ARGS);
+ File res =
+ ResultHandler.writeResults(
+ SUITE_NAME,
+ SUITE_VERSION,
+ SUITE_PLAN,
+ SUITE_BUILD,
+ result,
+ resultDir,
+ START_MS,
+ END_MS,
+ REFERENCE_URL,
+ LOG_URL,
+ COMMAND_LINE_ARGS);
+ String content = FileUtil.readStringFromFile(res);
+ assertXmlContainsAttribute(content, "Result/Build", "run_history", EXAMPLE_RUN_HISTORY);
+ assertXmlContainsNode(content, "Result/RunHistory");
+ assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000000000");
+ assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000000001");
+ assertXmlContainsAttribute(content, "Result/RunHistory/Run", "start", "10000000000002");
+ assertXmlContainsAttribute(content, "Result/RunHistory/Run", "end", "10000000000003");
// Parse the results and assert correctness
- checkResult(ResultHandler.getResultFromDir(resultDir), false);
+ result = ResultHandler.getResultFromDir(resultDir);
+ checkResult(result, false);
+ checkRunHistory(result);
+ }
+
+ /*
+ * Test serialization for CTS Verifier since test results with test result history is only in
+ * CTS Verifier and was not parsed by suite harness.
+ */
+ public void testSerialization_whenTestResultWithTestResultHistoryWithoutParsing()
+ throws Exception {
+ IInvocationResult result = new InvocationResult();
+ result.setStartTime(START_MS);
+ result.setTestPlan(SUITE_PLAN);
+ result.addDeviceSerial(DEVICE_A);
+ result.addDeviceSerial(DEVICE_B);
+ result.addInvocationInfo(BUILD_FINGERPRINT, EXAMPLE_BUILD_FINGERPRINT);
+ result.addInvocationInfo(BUILD_ID, EXAMPLE_BUILD_ID);
+ result.addInvocationInfo(BUILD_PRODUCT, EXAMPLE_BUILD_PRODUCT);
+
+ // Module A: test1 passes, test2 not executed
+ IModuleResult moduleA = result.getOrCreateModule(ID_A);
+ moduleA.setDone(false);
+ moduleA.addRuntime(Integer.parseInt(RUNTIME_A));
+ ICaseResult moduleACase = moduleA.getOrCreateResult(CLASS_A);
+ ITestResult moduleATest1 = moduleACase.getOrCreateResult(METHOD_1);
+ moduleATest1.setResultStatus(TestStatus.PASS);
+ // Module B: test3 fails with test result history, test4 passes with report log,
+ // test5 passes with skip
+ IModuleResult moduleB = result.getOrCreateModule(ID_B);
+ moduleB.setDone(true);
+ moduleB.addRuntime(Integer.parseInt(RUNTIME_B));
+ ICaseResult moduleBCase = moduleB.getOrCreateResult(CLASS_B);
+ Set<Map.Entry> durations = new HashSet<>();
+ durations.add(new AbstractMap.SimpleEntry<>(TEST_START_MS, TEST_END_MS));
+ ITestResult moduleBTest3 = moduleBCase.getOrCreateResult(METHOD_3);
+ moduleBTest3.setResultStatus(TestStatus.FAIL);
+ moduleBTest3.setMessage(MESSAGE);
+ moduleBTest3.setStackTrace(STACK_TRACE);
+ moduleBTest3.setBugReport(BUG_REPORT);
+ moduleBTest3.setLog(LOGCAT);
+ moduleBTest3.setScreenshot(SCREENSHOT);
+ List<TestResultHistory> resultHistories = new ArrayList<TestResultHistory>();
+ TestResultHistory resultHistory = new TestResultHistory(METHOD_3, durations);
+ resultHistories.add(resultHistory);
+ moduleBTest3.setTestResultHistories(resultHistories);
+ ITestResult moduleBTest4 = moduleBCase.getOrCreateResult(METHOD_4);
+ moduleBTest4.setResultStatus(TestStatus.PASS);
+ ReportLog report = new ReportLog();
+ ReportLog.Metric summary =
+ new ReportLog.Metric(
+ SUMMARY_SOURCE,
+ SUMMARY_MESSAGE,
+ SUMMARY_VALUE,
+ ResultType.HIGHER_BETTER,
+ ResultUnit.SCORE);
+ report.setSummary(summary);
+ moduleBTest4.setReportLog(report);
+ ITestResult moduleBTest5 = moduleBCase.getOrCreateResult(METHOD_5);
+ moduleBTest5.skipped();
+
+ // Serialize to file
+ File res =
+ ResultHandler.writeResults(
+ SUITE_NAME,
+ SUITE_VERSION,
+ SUITE_PLAN,
+ SUITE_BUILD,
+ result,
+ resultDir,
+ START_MS,
+ END_MS,
+ REFERENCE_URL,
+ LOG_URL,
+ COMMAND_LINE_ARGS);
+ String content = FileUtil.readStringFromFile(res);
+ assertXmlContainsNode(content, "Result/Module/TestCase/Test/RunHistory");
+ assertXmlContainsAttribute(
+ content,
+ "Result/Module/TestCase/Test/RunHistory/Run",
+ "start",
+ Long.toString(TEST_START_MS));
+ assertXmlContainsAttribute(
+ content,
+ "Result/Module/TestCase/Test/RunHistory/Run",
+ "end",
+ Long.toString(TEST_END_MS));
+ checkResult(result, EXAMPLE_BUILD_FINGERPRINT, false, false);
}
public void testParsing() throws Exception {
@@ -236,14 +372,16 @@
checkResult(
ResultHandler.getResultFromDir(resultDir),
EXAMPLE_BUILD_FINGERPRINT_UNALTERED,
- false);
+ false,
+ true);
}
public void testParsing_whenUnalteredBuildFingerprintIsEmpty_usesRegularBuildFingerprint() throws Exception {
String buildInfo = String.format(XML_BUILD_INFO_WITH_UNALTERED_BUILD_FINGERPRINT,
EXAMPLE_BUILD_FINGERPRINT, "", EXAMPLE_BUILD_ID, EXAMPLE_BUILD_PRODUCT);
File resultDir = writeResultDir(resultsDir, buildInfo, false);
- checkResult(ResultHandler.getResultFromDir(resultDir), EXAMPLE_BUILD_FINGERPRINT, false);
+ checkResult(
+ ResultHandler.getResultFromDir(resultDir), EXAMPLE_BUILD_FINGERPRINT, false, true);
}
public void testGetLightResults() throws Exception {
@@ -347,12 +485,17 @@
}
static void checkResult(IInvocationResult result, boolean newTestFormat) throws Exception {
- checkResult(result, EXAMPLE_BUILD_FINGERPRINT, newTestFormat);
+ checkResult(result, EXAMPLE_BUILD_FINGERPRINT, newTestFormat, true);
+ }
+
+ static void checkRunHistory(IInvocationResult result) {
+ Map<String, String> buildInfo = result.getInvocationInfo();
+ assertEquals("Incorrect run history", EXAMPLE_RUN_HISTORY, buildInfo.get(RUN_HISTORY));
}
static void checkResult(
- IInvocationResult result, String expectedBuildFingerprint, boolean newTestFormat)
- throws Exception {
+ IInvocationResult result, String expectedBuildFingerprint, boolean newTestFormat,
+ boolean checkResultHistories) throws Exception {
assertEquals("Expected 3 passes", 3, result.countResults(TestStatus.PASS));
assertEquals("Expected 1 failure", 1, result.countResults(TestStatus.FAIL));
@@ -418,6 +561,24 @@
assertEquals("Incorrect message", MESSAGE, moduleBTest3.getMessage());
assertEquals("Incorrect stack trace", STACK_TRACE, moduleBTest3.getStackTrace());
assertNull("Unexpected report", moduleBTest3.getReportLog());
+ List<TestResultHistory> resultHistories = moduleBTest3.getTestResultHistories();
+ // Check if unit tests do parsing result, because tests for CTS Verifier do not parse it.
+ if (checkResultHistories) {
+ // For xTS except CTS Verifier.
+ assertNull("Unexpected test result history list", resultHistories);
+ } else {
+ // For CTS Verifier.
+ assertNotNull("Expected test result history list", resultHistories);
+ assertEquals("Expected 1 test result history", 1, resultHistories.size());
+ for (TestResultHistory resultHistory : resultHistories) {
+ assertNotNull("Expected test result history", resultHistory);
+ assertEquals("Incorrect test name", METHOD_3, resultHistory.getTestName());
+ for (Map.Entry duration : resultHistory.getDurations()) {
+ assertEquals("Incorrect test start time", TEST_START_MS, duration.getKey());
+ assertEquals("Incorrect test end time", TEST_END_MS, duration.getValue());
+ }
+ }
+ }
ITestResult moduleBTest4 = moduleBResults.get(1);
assertEquals("Incorrect name", METHOD_4, moduleBTest4.getName());
assertEquals("Incorrect result", TestStatus.PASS, moduleBTest4.getResultStatus());
@@ -450,4 +611,55 @@
assertNull("Unexpected stack trace", moduleBTest5.getStackTrace());
assertNull("Unexpected report", moduleBTest5.getReportLog());
}
+
+ /** Return all XML nodes that match the given xPathExpression. */
+ private NodeList getXmlNodes(String xml, String xPathExpression)
+ throws XPathExpressionException {
+
+ InputSource inputSource = new InputSource(new StringReader(xml));
+ XPath xpath = XPathFactory.newInstance().newXPath();
+ return (NodeList) xpath.evaluate(xPathExpression, inputSource, XPathConstants.NODESET);
+ }
+
+ /** Assert that the XML contains a node matching the given xPathExpression. */
+ private NodeList assertXmlContainsNode(String xml, String xPathExpression)
+ throws XPathExpressionException {
+ NodeList nodes = getXmlNodes(xml, xPathExpression);
+ assertNotNull(
+ String.format("XML '%s' returned null for xpath '%s'.", xml, xPathExpression),
+ nodes);
+ assertTrue(
+ String.format(
+ "XML '%s' should have returned at least 1 node for xpath '%s', "
+ + "but returned %s nodes instead.",
+ xml, xPathExpression, nodes.getLength()),
+ nodes.getLength() >= 1);
+ return nodes;
+ }
+
+ /**
+ * Assert that the XML contains a node matching the given xPathExpression and that the node has
+ * a given value.
+ */
+ private void assertXmlContainsAttribute(
+ String xml, String xPathExpression, String attributeName, String attributeValue)
+ throws XPathExpressionException {
+ NodeList nodes = assertXmlContainsNode(xml, xPathExpression);
+ boolean found = false;
+
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+ String value = element.getAttribute(attributeName);
+ if (attributeValue.equals(value)) {
+ found = true;
+ break;
+ }
+ }
+
+ assertTrue(
+ String.format(
+ "xPath '%s' should contain attribute '%s' but does not. XML: '%s'",
+ xPathExpression, attributeName, xml),
+ found);
+ }
}
diff --git a/common/util/tests/src/com/android/compatibility/common/util/TestResultHistoryTest.java b/common/util/tests/src/com/android/compatibility/common/util/TestResultHistoryTest.java
new file mode 100644
index 0000000..740b084
--- /dev/null
+++ b/common/util/tests/src/com/android/compatibility/common/util/TestResultHistoryTest.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.compatibility.common.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.util.AbstractMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/** Unit tests for {@link TestResultHistory} */
+@RunWith(JUnit4.class)
+public class TestResultHistoryTest {
+
+ private static final String TEST_NAME = "Test";
+ private static final long START_TIME = 1000000000011L;
+ private static final long END_TIME = 1000000000012L;
+ private static final String HEADER_XML =
+ "<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>";
+ private static final String RUN_HISTORIES_XML =
+ HEADER_XML
+ + "\r\n"
+ + "<RunHistory>\r\n"
+ + " <Run start=\"1000000000011\" end=\"1000000000012\" />\r\n"
+ + "</RunHistory>";
+
+ private TestResultHistory mTestResultHistory;
+ private Set<Map.Entry> mDurations;
+
+ @Before
+ public void setUp() throws Exception {
+ mDurations = new HashSet<>();
+ }
+
+ @Test
+ public void testSerialize_null() throws Exception {
+ try {
+ TestResultHistory.serialize(null, null);
+ fail("Expected IllegalArgumentException when serializing an empty test result history");
+ } catch (IllegalArgumentException e) {
+ // Expected
+ }
+ }
+
+ @Test
+ public void testSerialize_full() throws Exception {
+ mDurations.add(new AbstractMap.SimpleEntry<>(START_TIME, END_TIME));
+ mTestResultHistory = new TestResultHistory(TEST_NAME, mDurations);
+ assertEquals(RUN_HISTORIES_XML, TestResultHistory.serialize(mTestResultHistory, TEST_NAME));
+ }
+}
diff --git a/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java b/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java
index a7f8460..7b1e053 100644
--- a/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java
+++ b/common/util/tests/src/com/android/compatibility/common/util/UnitTests.java
@@ -37,6 +37,7 @@
ReportLogTest.class,
ResultHandlerTest.class,
StatTest.class,
+ TestResultHistoryTest.class,
TestResultTest.class,
ReadElfTest.class,
})
diff --git a/tools/cts-instant-tradefed/tests/run_tests.sh b/tools/cts-instant-tradefed/tests/run_tests.sh
new file mode 100755
index 0000000..5ef31fc
--- /dev/null
+++ b/tools/cts-instant-tradefed/tests/run_tests.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Helper script for running unit tests for compatibility libraries
+
+# TODO: Change it to the new harness repo once harness code is moved
+# to a new repo.
+CTS_DIR=$(dirname ${0})/../../..
+source ${CTS_DIR}/test_defs.sh
+
+JARS="
+ compatibility-common-util-hostsidelib\
+ compatibility-host-util\
+ cts-instant-tradefed-tests\
+ cts-instant-tradefed"
+
+run_tests "com.android.compatibility.tradefed.CtsInstantTradefedTest" "${JARS}" "${@}"
diff --git a/tools/cts-tradefed/res/config/cts-known-failures.xml b/tools/cts-tradefed/res/config/cts-known-failures.xml
index 40b195b..88e549e 100755
--- a/tools/cts-tradefed/res/config/cts-known-failures.xml
+++ b/tools/cts-tradefed/res/config/cts-known-failures.xml
@@ -139,6 +139,10 @@
<!-- b/62481870 -->
<option name="compatibility:exclude-filter" value="CtsNativeMediaAAudioTestCases android.nativemedia.aaudio.AAudioOutputStreamCallbackTest#testPlayback" />
+ <!-- b/134654621 -->
+ <option name="compatibility:exclude-filter" value="CtsWindowManagerDeviceTestCases android.server.wm.AppConfigurationTests#testTaskCloseRestoreFreeOrientation" />
+ <option name="compatibility:exclude-filter" value="CtsWindowManagerDeviceTestCases android.server.wm.AppConfigurationTests#testAppOrientationRequestConfigClears" />
+
<!-- b/62976713 -->
<option name="compatibility:exclude-filter" value="arm64-v8a CtsMediaBitstreamsTestCases" />
<option name="compatibility:exclude-filter" value="x86_64 CtsMediaBitstreamsTestCases" />
@@ -179,8 +183,6 @@
<!-- fails only on angler/bullhead userdebug -->
<option name="compatibility:exclude-filter" value="CtsLiblogTestCases liblog#wrap_mode_blocks" />
- <!-- b/80309889 -->
- <option name="compatibility:exclude-filter" value="CtsStatsdHostTestCases android.cts.statsd.atom.UidAtomTests#testCpuActiveTime" />
<!-- b/132274449 -->
<option name="compatibility:exclude-filter" value="CtsStatsdHostTestCases android.cts.statsd.validation.BatteryStatsValidationTests#testConnectivityStateChange" />
@@ -205,7 +207,4 @@
<!-- b/129859594 -->
<option name="compatibility:exclude-filter" value="CtsAtomicInstallTestCases com.android.tests.atomicinstall.AtomicInstallTest#testFailInconsistentMultiPackageCommit" />
-
- <!-- b/126548816 -->
- <option name="compatibility:exclude-filter" value="CtsRcsTestCases" />
</configuration>