| /* |
| * Copyright (C) 2016 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.android.compatibility.testtype; |
| |
| import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; |
| import com.android.compatibility.common.util.AbiUtils; |
| import com.android.ddmlib.IShellOutputReceiver; |
| import com.android.ddmlib.Log; |
| import com.android.ddmlib.Log.LogLevel; |
| import com.android.ddmlib.MultiLineReceiver; |
| import com.android.ddmlib.testrunner.TestIdentifier; |
| import com.android.tradefed.build.IBuildInfo; |
| import com.android.tradefed.config.Option; |
| import com.android.tradefed.config.OptionCopier; |
| import com.android.tradefed.device.DeviceNotAvailableException; |
| import com.android.tradefed.device.ITestDevice; |
| import com.android.tradefed.result.ITestInvocationListener; |
| import com.android.tradefed.testtype.IAbi; |
| import com.android.tradefed.testtype.IAbiReceiver; |
| import com.android.tradefed.testtype.IBuildReceiver; |
| import com.android.tradefed.testtype.IDeviceTest; |
| import com.android.tradefed.testtype.IRemoteTest; |
| import com.android.tradefed.testtype.IRuntimeHintProvider; |
| import com.android.tradefed.testtype.IShardableTest; |
| import com.android.tradefed.testtype.ITestCollector; |
| import com.android.tradefed.testtype.ITestFileFilterReceiver; |
| import com.android.tradefed.testtype.ITestFilterReceiver; |
| import com.android.tradefed.util.ArrayUtil; |
| import com.android.tradefed.util.FileUtil; |
| import com.android.tradefed.util.TimeVal; |
| import com.google.common.base.Splitter; |
| |
| import vogar.ExpectationStore; |
| import vogar.ModeId; |
| |
| import java.io.BufferedReader; |
| import java.io.File; |
| import java.io.FilenameFilter; |
| import java.io.FileReader; |
| import java.io.IOException; |
| import java.io.PrintWriter; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Set; |
| import java.util.concurrent.TimeUnit; |
| |
| /** |
| * A wrapper to run tests against Dalvik. |
| */ |
| public class DalvikTest implements IAbiReceiver, IBuildReceiver, IDeviceTest, IRemoteTest, |
| IRuntimeHintProvider, IShardableTest, ITestCollector, ITestFileFilterReceiver, |
| ITestFilterReceiver { |
| |
| private static final String TAG = DalvikTest.class.getSimpleName(); |
| |
| /** |
| * TEST_PACKAGES is a Set containing the names of packages on the classpath known to contain |
| * tests to be run under DalvikTest. The TEST_PACKAGES set is used to shard DalvikTest into |
| * multiple DalvikTests, each responsible for running one of these packages' tests. |
| */ |
| private static final Set<String> TEST_PACKAGES = new HashSet<>(); |
| private static final String JDWP_PACKAGE_BASE = "org.apache.harmony.jpda.tests.jdwp.%s"; |
| static { |
| // Though uppercase, these are package names, not class names |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ArrayReference")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ArrayType")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ClassLoaderReference")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ClassObjectReference")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ClassType")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "DebuggerOnDemand")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "Deoptimization")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "EventModifiers")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "Events")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "InterfaceType")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "Method")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "MultiSession")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ObjectReference")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ReferenceType")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "StackFrame")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "StringReference")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ThreadGroupReference")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "ThreadReference")); |
| TEST_PACKAGES.add(String.format(JDWP_PACKAGE_BASE, "VirtualMachine")); |
| } |
| |
| private static final String EXPECTATIONS_EXT = ".expectations"; |
| // Command to run the VM, args are bitness, classpath, dalvik-args, abi, runner-args, |
| // include and exclude filters, and exclude filters file. |
| private static final String COMMAND = "dalvikvm%s -classpath %s %s " |
| + "com.android.compatibility.dalvik.DalvikTestRunner --abi=%s %s %s %s %s %s %s"; |
| private static final String INCLUDE_FILE = "/data/local/tmp/dalvik/includes"; |
| private static final String EXCLUDE_FILE = "/data/local/tmp/dalvik/excludes"; |
| private static String START_RUN = "start-run"; |
| private static String END_RUN = "end-run"; |
| private static String START_TEST = "start-test"; |
| private static String END_TEST = "end-test"; |
| private static String FAILURE = "failure"; |
| |
| @Option(name = "run-name", description = "The name to use when reporting results") |
| private String mRunName; |
| |
| @Option(name = "classpath", description = "Holds the paths to search when loading tests") |
| private List<String> mClasspath = new ArrayList<>(); |
| |
| @Option(name = "dalvik-arg", description = "Holds arguments to pass to Dalvik") |
| private List<String> mDalvikArgs = new ArrayList<>(); |
| |
| @Option(name = "runner-arg", |
| description = "Holds arguments to pass to the device-side test runner") |
| private List<String> mRunnerArgs = new ArrayList<>(); |
| |
| @Option(name = "include-filter", |
| description = "The include filters of the test name to run.") |
| private List<String> mIncludeFilters = new ArrayList<>(); |
| |
| @Option(name = "exclude-filter", |
| description = "The exclude filters of the test name to run.") |
| private List<String> mExcludeFilters = new ArrayList<>(); |
| |
| @Option(name = "test-file-include-filter", |
| description="A file containing a list of line separated test classes and optionally" |
| + " methods to include") |
| private File mIncludeTestFile = null; |
| |
| @Option(name = "test-file-exclude-filter", |
| description="A file containing a list of line separated test classes and optionally" |
| + " methods to exclude") |
| private File mExcludeTestFile = null; |
| |
| @Option(name = "runtime-hint", |
| isTimeVal = true, |
| description="The hint about the test's runtime.") |
| private long mRuntimeHint = 60000;// 1 minute |
| |
| @Option(name = "known-failures", |
| description = "Comma-separated list of files specifying known-failures to be skipped") |
| private String mKnownFailures; |
| |
| @Option(name = "collect-tests-only", |
| description = "Only invoke the instrumentation to collect list of applicable test " |
| + "cases. All test run callbacks will be triggered, but test execution will " |
| + "not be actually carried out.") |
| private boolean mCollectTestsOnly = false; |
| |
| @Option(name = "per-test-timeout", |
| description = "The maximum amount of time during which the DalvikTestRunner may " |
| + "yield no output. Because the runner outputs results for each test, this " |
| + "is essentially a per-test timeout") |
| private long mPerTestTimeout = 10; // 10 minutes |
| |
| private IAbi mAbi; |
| private CompatibilityBuildHelper mBuildHelper; |
| private ITestDevice mDevice; |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void setAbi(IAbi abi) { |
| mAbi = abi; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void setBuild(IBuildInfo build) { |
| mBuildHelper = new CompatibilityBuildHelper(build); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void setDevice(ITestDevice device) { |
| mDevice = device; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public ITestDevice getDevice() { |
| return mDevice; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void addIncludeFilter(String filter) { |
| mIncludeFilters.add(filter); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void addAllIncludeFilters(Set<String> filters) { |
| mIncludeFilters.addAll(filters); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void addExcludeFilter(String filter) { |
| mExcludeFilters.add(filter); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void addAllExcludeFilters(Set<String> filters) { |
| mExcludeFilters.addAll(filters); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void setIncludeTestFile(File testFile) { |
| mIncludeTestFile = testFile; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void setExcludeTestFile(File testFile) { |
| mExcludeTestFile = testFile; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public long getRuntimeHint() { |
| return mRuntimeHint; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void setCollectTestsOnly(boolean shouldCollectTest) { |
| mCollectTestsOnly = shouldCollectTest; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void run(final ITestInvocationListener listener) throws DeviceNotAvailableException { |
| String abiName = mAbi.getName(); |
| String bitness = AbiUtils.getBitness(abiName); |
| |
| File tmpExcludeFile = null; |
| try { |
| // push one file of exclude filters to the device |
| tmpExcludeFile = getExcludeFile(); |
| if (!mDevice.pushFile(tmpExcludeFile, EXCLUDE_FILE)) { |
| Log.logAndDisplay(LogLevel.ERROR, TAG, "Couldn't push file: " + tmpExcludeFile); |
| } |
| } catch (IOException e) { |
| throw new RuntimeException("Failed to parse expectations", e); |
| } finally { |
| FileUtil.deleteFile(tmpExcludeFile); |
| } |
| |
| // push one file of include filters to the device, if file exists |
| if (mIncludeTestFile != null) { |
| String path = mIncludeTestFile.getAbsolutePath(); |
| if (!mIncludeTestFile.isFile() || !mIncludeTestFile.canRead()) { |
| throw new RuntimeException(String.format("Failed to read include file %s", path)); |
| } |
| if (!mDevice.pushFile(mIncludeTestFile, INCLUDE_FILE)) { |
| Log.logAndDisplay(LogLevel.ERROR, TAG, "Couldn't push file: " + path); |
| } |
| } |
| |
| |
| // Create command |
| mDalvikArgs.add("-Duser.name=shell"); |
| mDalvikArgs.add("-Duser.language=en"); |
| mDalvikArgs.add("-Duser.region=US"); |
| mDalvikArgs.add("-Xcheck:jni"); |
| mDalvikArgs.add("-Xjnigreflimit:2000"); |
| |
| String dalvikArgs = ArrayUtil.join(" ", mDalvikArgs); |
| dalvikArgs = dalvikArgs.replace("|#ABI#|", bitness); |
| |
| String runnerArgs = ArrayUtil.join(" ", mRunnerArgs); |
| // Filters |
| StringBuilder includeFilters = new StringBuilder(); |
| if (!mIncludeFilters.isEmpty()) { |
| includeFilters.append("--include-filter="); |
| includeFilters.append(ArrayUtil.join(",", mIncludeFilters)); |
| } |
| StringBuilder excludeFilters = new StringBuilder(); |
| if (!mExcludeFilters.isEmpty()) { |
| excludeFilters.append("--exclude-filter="); |
| excludeFilters.append(ArrayUtil.join(",", mExcludeFilters)); |
| } |
| // Filter files |
| String includeFile = String.format("--include-filter-file=%s", INCLUDE_FILE); |
| String excludeFile = String.format("--exclude-filter-file=%s", EXCLUDE_FILE); |
| // Communicate with DalvikTestRunner if tests should only be collected |
| String collectTestsOnlyString = (mCollectTestsOnly) ? "--collect-tests-only" : ""; |
| final String command = String.format(COMMAND, bitness, |
| ArrayUtil.join(File.pathSeparator, mClasspath), |
| dalvikArgs, abiName, runnerArgs, |
| includeFilters, excludeFilters, includeFile, excludeFile, collectTestsOnlyString); |
| IShellOutputReceiver receiver = new MultiLineReceiver() { |
| private TestIdentifier test; |
| |
| @Override |
| public boolean isCancelled() { |
| return false; |
| } |
| |
| @Override |
| public void processNewLines(String[] lines) { |
| for (String line : lines) { |
| String[] parts = line.split(":", 2); |
| String tag = parts[0]; |
| if (tag.equals(START_RUN)) { |
| listener.testRunStarted(mRunName, Integer.parseInt(parts[1])); |
| Log.logAndDisplay(LogLevel.INFO, TAG, command); |
| Log.logAndDisplay(LogLevel.INFO, TAG, line); |
| } else if (tag.equals(END_RUN)) { |
| listener.testRunEnded(Integer.parseInt(parts[1]), |
| Collections.<String, String>emptyMap()); |
| Log.logAndDisplay(LogLevel.INFO, TAG, line); |
| } else if (tag.equals(START_TEST)) { |
| test = getTestIdentifier(parts[1]); |
| listener.testStarted(test); |
| } else if (tag.equals(FAILURE)) { |
| listener.testFailed(test, parts[1]); |
| } else if (tag.equals(END_TEST)) { |
| listener.testEnded(getTestIdentifier(parts[1]), |
| Collections.<String, String>emptyMap()); |
| } else { |
| Log.logAndDisplay(LogLevel.INFO, TAG, line); |
| } |
| } |
| } |
| |
| private TestIdentifier getTestIdentifier(String name) { |
| String[] parts = name.split("#"); |
| String className = parts[0]; |
| String testName = ""; |
| if (parts.length > 1) { |
| testName = parts[1]; |
| } |
| return new TestIdentifier(className, testName); |
| } |
| |
| }; |
| mDevice.executeShellCommand(command, receiver, mPerTestTimeout, TimeUnit.MINUTES, 1); |
| } |
| |
| /* |
| * Due to known failures, there are typically too many excludes to pass via command line. |
| * Collect excludes from .expectation files in the testcases directory, from files in the |
| * module's resources directory, and from mExcludeTestFile, if set. |
| */ |
| private File getExcludeFile() throws IOException { |
| File excludeFile = null; |
| PrintWriter out = null; |
| |
| try { |
| excludeFile = File.createTempFile("excludes", "txt"); |
| out = new PrintWriter(excludeFile); |
| // create expectation store from set of expectation files found in testcases dir |
| Set<File> expectationFiles = new HashSet<>(); |
| for (File f : mBuildHelper.getTestsDir().listFiles( |
| new ExpectationFileFilter(mRunName))) { |
| expectationFiles.add(f); |
| } |
| ExpectationStore testsDirStore = |
| ExpectationStore.parse(expectationFiles, ModeId.DEVICE); |
| // create expectation store from expectation files found in module resources dir |
| ExpectationStore resourceStore = null; |
| if (mKnownFailures != null) { |
| Splitter splitter = Splitter.on(',').trimResults(); |
| Set<String> knownFailuresFiles = |
| new HashSet<>(splitter.splitToList(mKnownFailures)); |
| resourceStore = ExpectationStore.parseResources( |
| getClass(), knownFailuresFiles, ModeId.DEVICE); |
| } |
| // Add expectations from testcases dir |
| for (String exclude : testsDirStore.getAllFailures().keySet()) { |
| out.println(exclude); |
| } |
| for (String exclude : testsDirStore.getAllOutComes().keySet()) { |
| out.println(exclude); |
| } |
| // Add expectations from resources dir |
| if (resourceStore != null) { |
| for (String exclude : resourceStore.getAllFailures().keySet()) { |
| out.println(exclude); |
| } |
| for (String exclude : resourceStore.getAllOutComes().keySet()) { |
| out.println(exclude); |
| } |
| } |
| // Add excludes from test-file-exclude-filter option |
| for (String exclude : getFiltersFromFile(mExcludeTestFile)) { |
| out.println(exclude); |
| } |
| out.flush(); |
| } finally { |
| if (out != null) { |
| out.close(); |
| } |
| } |
| return excludeFile; |
| } |
| |
| |
| /* |
| * Helper method that reads filters from a file into a set. |
| * Returns an empty set given a null file |
| */ |
| private static Set<String> getFiltersFromFile(File f) throws IOException { |
| Set<String> filters = new HashSet<String>(); |
| if (f != null) { |
| BufferedReader reader = new BufferedReader(new FileReader(f)); |
| String filter = null; |
| while ((filter = reader.readLine()) != null) { |
| filters.add(filter); |
| } |
| reader.close(); |
| } |
| return filters; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public Collection<IRemoteTest> split() { |
| List<IRemoteTest> shards = new ArrayList<>(); |
| // A DalvikTest to run any tests not contained in packages from TEST_PACKAGES, may be empty |
| DalvikTest catchAll = new DalvikTest(); |
| OptionCopier.copyOptionsNoThrow(this, catchAll); |
| shards.add(catchAll); |
| // estimate catchAll's runtime to be that of a single package in TEST_PACKAGES |
| long runtimeHint = mRuntimeHint / TEST_PACKAGES.size(); |
| catchAll.mRuntimeHint = runtimeHint; |
| for (String packageName: TEST_PACKAGES) { |
| catchAll.addExcludeFilter(packageName); |
| // create one shard for package 'packageName' |
| DalvikTest test = new DalvikTest(); |
| OptionCopier.copyOptionsNoThrow(this, test); |
| test.addIncludeFilter(packageName); |
| test.mRuntimeHint = runtimeHint; |
| shards.add(test); |
| } |
| // return a shard for each package in TEST_PACKAGE, plus a shard for any other tests |
| return shards; |
| } |
| |
| /** |
| * A {@link FilenameFilter} to find all the expectation files in a directory. |
| */ |
| public static class ExpectationFileFilter implements FilenameFilter { |
| |
| private String mName; |
| |
| public ExpectationFileFilter(String name) { |
| mName = name; |
| } |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public boolean accept(File dir, String name) { |
| return name.startsWith(mName) && name.endsWith(EXPECTATIONS_EXT); |
| } |
| } |
| } |