blob: f0031604215bec3ccbe93bc7a1ff60a23d0b7b77 [file] [log] [blame]
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.tradefed.testtype;
import com.android.ddmlib.IShellOutputReceiver;
import com.android.tradefed.config.IConfiguration;
import com.android.tradefed.config.IConfigurationReceiver;
import com.android.tradefed.config.Option;
import com.android.tradefed.config.OptionCopier;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.testtype.coverage.CoverageOptions;
import com.android.tradefed.util.ArrayUtil;
import com.google.common.annotations.VisibleForTesting;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
/** The base class of gTest */
public abstract class GTestBase
implements IRemoteTest,
IConfigurationReceiver,
ITestFilterReceiver,
IRuntimeHintProvider,
ITestCollector,
IShardableTest {
private static final List<String> DEFAULT_FILE_EXCLUDE_FILTERS = new ArrayList<>();
static {
// Exclude .so by default as they are not runnable.
DEFAULT_FILE_EXCLUDE_FILTERS.add(".*\\.so");
}
@Option(name = "run-disable-tests", description = "Determine to run disable tests or not.")
private boolean mRunDisabledTests = false;
@Option(name = "module-name", description = "The name of the native test module to run.")
private String mTestModule = null;
@Option(
name = "file-exclusion-filter-regex",
description = "Regex to exclude certain files from executing. Can be repeated")
private List<String> mFileExclusionFilterRegex = new ArrayList<>(DEFAULT_FILE_EXCLUDE_FILTERS);
@Option(
name = "positive-testname-filter",
description = "The GTest-based positive filter of the test name to run.")
private String mTestNamePositiveFilter = null;
@Option(
name = "negative-testname-filter",
description = "The GTest-based negative filter of the test name to run.")
private String mTestNameNegativeFilter = null;
@Option(
name = "include-filter",
description = "The GTest-based positive filter of the test names to run."
)
private Set<String> mIncludeFilters = new LinkedHashSet<>();
@Option(
name = "exclude-filter",
description = "The GTest-based negative filter of the test names to run."
)
private Set<String> mExcludeFilters = new LinkedHashSet<>();
@Option(
name = "native-test-timeout",
description =
"The max time for a gtest to run. Test run will be aborted if any test "
+ "takes longer.",
isTimeVal = true)
private long mMaxTestTimeMs = 1 * 60 * 1000L;
/** @deprecated use --coverage in CoverageOptions instead. */
@Deprecated
@Option(
name = "coverage",
description =
"Collect code coverage for this test run. Note that the build under test must be a "
+ "coverage build or else this will fail."
)
private boolean mCoverage = false;
@Option(
name = "prepend-filename",
description = "Prepend filename as part of the classname for the tests.")
private boolean mPrependFileName = false;
@Option(name = "before-test-cmd", description = "adb shell command(s) to run before GTest.")
private List<String> mBeforeTestCmd = new ArrayList<>();
@Option(name = "after-test-cmd", description = "adb shell command(s) to run after GTest.")
private List<String> mAfterTestCmd = new ArrayList<>();
@Option(name = "run-test-as", description = "User to execute test binary as.")
private String mRunTestAs = null;
@Option(
name = "ld-library-path",
description = "LD_LIBRARY_PATH value to include in the GTest execution command.")
private String mLdLibraryPath = null;
@Option(
name = "native-test-flag",
description =
"Additional flag values to pass to the native test's shell command. "
+ "Flags should be complete, including any necessary dashes: \"--flag=value\"")
private List<String> mGTestFlags = new ArrayList<>();
@Option(
name = "runtime-hint",
description = "The hint about the test's runtime.",
isTimeVal = true)
private long mRuntimeHint = 60000; // 1 minute
@Option(
name = "xml-output",
description =
"Use gtest xml output for test results, "
+ "if test binaries crash, no output will be available.")
private boolean mEnableXmlOutput = false;
@Option(
name = "collect-tests-only",
description =
"Only invoke the test binary to collect list of applicable test cases. "
+ "All test run callbacks will be triggered, but test execution will "
+ "not be actually carried out. This option ignores sharding parameters, so "
+ "each shard will end up collecting all tests.")
private boolean mCollectTestsOnly = false;
@Option(
name = "test-filter-key",
description =
"run the gtest with the --gtest_filter populated with the filter from "
+ "the json filter file associated with the binary, the filter file will have "
+ "the same name as the binary with the .json extension.")
private String mTestFilterKey = null;
// GTest flags...
protected static final String GTEST_FLAG_PRINT_TIME = "--gtest_print_time";
protected static final String GTEST_FLAG_FILTER = "--gtest_filter";
protected static final String GTEST_FLAG_RUN_DISABLED_TESTS = "--gtest_also_run_disabled_tests";
protected static final String GTEST_FLAG_LIST_TESTS = "--gtest_list_tests";
protected static final String GTEST_XML_OUTPUT = "--gtest_output=xml:%s";
// Expected extension for the filter file associated with the binary (json formatted file)
@VisibleForTesting protected static final String FILTER_EXTENSION = ".filter";
private int mShardCount = 0;
private int mShardIndex = 0;
private boolean mIsSharded = false;
private IConfiguration mConfiguration = null;
/** {@inheritDoc} */
@Override
public void setConfiguration(IConfiguration configuration) {
mConfiguration = configuration;
}
/**
* Set the Android native test module to run.
*
* @param moduleName The name of the native test module to run
*/
public void setModuleName(String moduleName) {
mTestModule = moduleName;
}
/**
* Get the Android native test module to run.
*
* @return the name of the native test module to run, or null if not set
*/
public String getModuleName() {
return mTestModule;
}
/** Set whether GTest should run disabled tests. */
protected void setRunDisabled(boolean runDisabled) {
mRunDisabledTests = runDisabled;
}
/**
* Get whether GTest should run disabled tests.
*
* @return True if disabled tests should be run, false otherwise
*/
public boolean getRunDisabledTests() {
return mRunDisabledTests;
}
/** Set the max time in ms for a gtest to run. */
@VisibleForTesting
void setMaxTestTimeMs(int timeout) {
mMaxTestTimeMs = timeout;
}
/**
* Adds an exclusion file filter regex.
*
* @param regex to exclude file.
*/
@VisibleForTesting
void addFileExclusionFilterRegex(String regex) {
mFileExclusionFilterRegex.add(regex);
}
/** Sets the shard index of this test. */
public void setShardIndex(int shardIndex) {
mShardIndex = shardIndex;
}
/** Gets the shard index of this test. */
public int getShardIndex() {
return mShardIndex;
}
/** Sets the shard count of this test. */
public void setShardCount(int shardCount) {
mShardCount = shardCount;
}
/** Returns the current shard-count. */
public int getShardCount() {
return mShardCount;
}
/** {@inheritDoc} */
@Override
public long getRuntimeHint() {
return mRuntimeHint;
}
/** {@inheritDoc} */
@Override
public void addIncludeFilter(String filter) {
if (mShardCount > 0) {
// If we explicitly start giving filters to GTest, reset the shard-count. GTest first
// applies filters then GTEST_TOTAL_SHARDS so it will probably end up not running
// anything
mShardCount = 0;
}
mIncludeFilters.add(cleanFilter(filter));
}
/** {@inheritDoc} */
@Override
public void addAllIncludeFilters(Set<String> filters) {
for (String filter : filters) {
mIncludeFilters.add(cleanFilter(filter));
}
}
/** {@inheritDoc} */
@Override
public void addExcludeFilter(String filter) {
mExcludeFilters.add(cleanFilter(filter));
}
/** {@inheritDoc} */
@Override
public void addAllExcludeFilters(Set<String> filters) {
for (String filter : filters) {
mExcludeFilters.add(cleanFilter(filter));
}
}
/** {@inheritDoc} */
@Override
public void clearIncludeFilters() {
mIncludeFilters.clear();
}
/** {@inheritDoc} */
@Override
public Set<String> getIncludeFilters() {
return mIncludeFilters;
}
/** {@inheritDoc} */
@Override
public Set<String> getExcludeFilters() {
return mExcludeFilters;
}
/** {@inheritDoc} */
@Override
public void clearExcludeFilters() {
mExcludeFilters.clear();
}
/** Gets module name. */
public String getTestModule() {
return mTestModule;
}
/** Gets regex to exclude certain files from executing. */
public List<String> getFileExclusionFilterRegex() {
return mFileExclusionFilterRegex;
}
/** Gets the max time for a gtest to run. */
public long getMaxTestTimeMs() {
return mMaxTestTimeMs;
}
/** Gets shell command(s) to run before GTest. */
public List<String> getBeforeTestCmd() {
return mBeforeTestCmd;
}
/** Gets shell command(s) to run after GTest. */
public List<String> getAfterTestCmd() {
return mAfterTestCmd;
}
/** Gets Additional flag values to pass to the native test's shell command. */
public List<String> getGTestFlags() {
return mGTestFlags;
}
/** Gets test filter key. */
public String getTestFilterKey() {
return mTestFilterKey;
}
/** Gets use gtest xml output for test results or not. */
public boolean isEnableXmlOutput() {
return mEnableXmlOutput;
}
/** Gets only invoke the test binary to collect list of applicable test cases or not. */
public boolean isCollectTestsOnly() {
return mCollectTestsOnly;
}
/** Gets isSharded flag. */
public boolean isSharded() {
return mIsSharded;
}
/**
* Define get filter method.
*
* <p>Sub class must implement how to get it's own filter.
*
* @param path the full path of the filter file.
* @return filter string.
*/
protected abstract String loadFilter(String path) throws DeviceNotAvailableException;
/**
* Helper to get the g-test filter of test to run.
*
* <p>Note that filters filter on the function name only (eg: Google Test "Test"); all Google
* Test "Test Cases" will be considered.
*
* @param path the full path of the binary on the device.
* @return the full filter flag to pass to the g-test, or an empty string if none have been
* specified
*/
protected String getGTestFilters(String path) throws DeviceNotAvailableException {
StringBuilder filter = new StringBuilder();
if (mTestNamePositiveFilter != null) {
mIncludeFilters.add(mTestNamePositiveFilter);
}
if (mTestNameNegativeFilter != null) {
mExcludeFilters.add(mTestNameNegativeFilter);
}
if (mTestFilterKey != null) {
if (!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) {
CLog.w("Using json file filter, --include/exclude-filter will be ignored.");
}
String fileFilters = loadFilter(path);
if (fileFilters != null && !fileFilters.isEmpty()) {
filter.append(GTEST_FLAG_FILTER);
filter.append("=");
filter.append(fileFilters);
}
} else {
if (!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) {
filter.append(GTEST_FLAG_FILTER);
filter.append("=");
if (!mIncludeFilters.isEmpty()) {
filter.append(ArrayUtil.join(":", mIncludeFilters));
}
if (!mExcludeFilters.isEmpty()) {
filter.append("-");
filter.append(ArrayUtil.join(":", mExcludeFilters));
}
}
}
return filter.toString();
}
/**
* Helper to get all the GTest flags to pass into the adb shell command.
*
* @param path the full path of the binary on the device.
* @return the {@link String} of all the GTest flags that should be passed to the GTest
*/
protected String getAllGTestFlags(String path) throws DeviceNotAvailableException {
String flags = String.format("%s %s", GTEST_FLAG_PRINT_TIME, getGTestFilters(path));
if (getRunDisabledTests()) {
flags = String.format("%s %s", flags, GTEST_FLAG_RUN_DISABLED_TESTS);
}
if (isCollectTestsOnly()) {
flags = String.format("%s %s", flags, GTEST_FLAG_LIST_TESTS);
}
for (String gTestFlag : getGTestFlags()) {
flags = String.format("%s %s", flags, gTestFlag);
}
return flags;
}
/*
* Conforms filters using a {@link TestDescription} format to be recognized by the GTest
* executable.
*/
public String cleanFilter(String filter) {
return filter.replace('#', '.');
}
/**
* Exposed for testing
*
* @param testRunName
* @param listener
* @return a {@link GTestXmlResultParser}
*/
@VisibleForTesting
GTestXmlResultParser createXmlParser(String testRunName, ITestInvocationListener listener) {
return new GTestXmlResultParser(testRunName, listener);
}
/**
* Factory method for creating a {@link IShellOutputReceiver} that parses test output and
* forwards results to the result listener.
*
* @param listener
* @param runName
* @return a {@link IShellOutputReceiver}
*/
@VisibleForTesting
IShellOutputReceiver createResultParser(String runName, ITestInvocationListener listener) {
IShellOutputReceiver receiver = null;
if (mCollectTestsOnly) {
GTestListTestParser resultParser = new GTestListTestParser(runName, listener);
resultParser.setPrependFileName(mPrependFileName);
receiver = resultParser;
} else {
GTestResultParser resultParser = new GTestResultParser(runName, listener);
resultParser.setPrependFileName(mPrependFileName);
receiver = resultParser;
}
// Erase the prepended binary name if needed
erasePrependedFileName(mExcludeFilters, runName);
erasePrependedFileName(mIncludeFilters, runName);
return receiver;
}
/**
* Helper method to build the gtest command to run.
*
* @param fullPath absolute file system path to gtest binary on device
* @param flags gtest execution flags
* @return the shell command line to run for the gtest
*/
protected String getGTestCmdLine(String fullPath, String flags) {
StringBuilder gTestCmdLine = new StringBuilder();
if (mLdLibraryPath != null) {
gTestCmdLine.append(String.format("LD_LIBRARY_PATH=%s ", mLdLibraryPath));
}
if (getCoverageOptions().isCoverageEnabled()) {
gTestCmdLine.append("GCOV_PREFIX=/data/misc/trace/testcoverage ");
}
// su to requested user
if (mRunTestAs != null) {
gTestCmdLine.append(String.format("su %s ", mRunTestAs));
}
gTestCmdLine.append(String.format("%s %s", fullPath, flags));
return gTestCmdLine.toString();
}
/** {@inheritDoc} */
@Override
public void setCollectTestsOnly(boolean shouldCollectTest) {
mCollectTestsOnly = shouldCollectTest;
}
/** {@inheritDoc} */
@Override
public Collection<IRemoteTest> split(int shardCountHint) {
if (shardCountHint <= 1 || mIsSharded) {
return null;
}
if (mCollectTestsOnly) {
// GTest cannot shard and use collect tests only, so prevent sharding in this case.
return null;
}
Collection<IRemoteTest> tests = new ArrayList<>();
for (int i = 0; i < shardCountHint; i++) {
tests.add(getTestShard(shardCountHint, i));
}
return tests;
}
/**
* Make a best effort attempt to retrieve a meaningful short descriptive message for given
* {@link Exception}
*
* @param e the {@link Exception}
* @return a short message
*/
protected String getExceptionMessage(Exception e) {
StringBuilder msgBuilder = new StringBuilder();
if (e.getMessage() != null) {
msgBuilder.append(e.getMessage());
}
if (e.getCause() != null) {
msgBuilder.append(" cause:");
msgBuilder.append(e.getCause().getClass().getSimpleName());
if (e.getCause().getMessage() != null) {
msgBuilder.append(" (");
msgBuilder.append(e.getCause().getMessage());
msgBuilder.append(")");
}
}
return msgBuilder.toString();
}
protected void erasePrependedFileName(Set<String> filters, String filename) {
if (!mPrependFileName) {
return;
}
Set<String> copy = new LinkedHashSet<>();
for (String filter : filters) {
if (filter.startsWith(filename + ".")) {
copy.add(filter.substring(filename.length() + 1));
} else {
copy.add(filter);
}
}
filters.clear();
filters.addAll(copy);
}
private IRemoteTest getTestShard(int shardCount, int shardIndex) {
GTestBase shard = null;
try {
shard = this.getClass().newInstance();
OptionCopier.copyOptionsNoThrow(this, shard);
shard.mShardIndex = shardIndex;
shard.mShardCount = shardCount;
shard.mIsSharded = true;
// We approximate the runtime of each shard to be equal since we can't know.
shard.mRuntimeHint = mRuntimeHint / shardCount;
} catch (InstantiationException | IllegalAccessException e) {
// This cannot happen because the class was already created once at that point.
throw new RuntimeException(
String.format(
"%s (%s) when attempting to create shard object",
e.getClass().getSimpleName(), getExceptionMessage(e)));
}
return shard;
}
/**
* Returns the {@link CoverageOptions} for this test, if it exists. Otherwise returns a default
* {@link CoverageOptions} object with all coverage disabled.
*/
protected CoverageOptions getCoverageOptions() {
if (mConfiguration != null) {
return mConfiguration.getCoverageOptions();
}
return new CoverageOptions();
}
}