Merge "Support sharding with test information"
diff --git a/invocation_interfaces/com/android/tradefed/invoker/ExecutionFiles.java b/invocation_interfaces/com/android/tradefed/invoker/ExecutionFiles.java
index 09bd185..55e0a33 100644
--- a/invocation_interfaces/com/android/tradefed/invoker/ExecutionFiles.java
+++ b/invocation_interfaces/com/android/tradefed/invoker/ExecutionFiles.java
@@ -182,6 +182,7 @@
continue;
}
FileUtil.recursiveDelete(mFiles.get(key));
+ mFiles.remove(key);
}
}
}
diff --git a/src/com/android/tradefed/invoker/TestInvocation.java b/src/com/android/tradefed/invoker/TestInvocation.java
index 44063bd..717a2a1 100644
--- a/src/com/android/tradefed/invoker/TestInvocation.java
+++ b/src/com/android/tradefed/invoker/TestInvocation.java
@@ -40,7 +40,9 @@
import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey;
import com.android.tradefed.invoker.sandbox.ParentSandboxInvocationExecution;
import com.android.tradefed.invoker.sandbox.SandboxedInvocationExecution;
+import com.android.tradefed.invoker.shard.LastShardDetector;
import com.android.tradefed.invoker.shard.ShardBuildCloner;
+import com.android.tradefed.invoker.shard.ShardHelper;
import com.android.tradefed.log.BaseLeveledLogOutput;
import com.android.tradefed.log.ILeveledLogOutput;
import com.android.tradefed.log.ILogRegistry;
@@ -279,7 +281,7 @@
// under certain cases it might still be possible to grab a bugreport
bugreportName = DEVICE_UNRESPONSIVE_BUGREPORT_NAME;
}
- resumed = resume(config, context, rescheduler, System.currentTimeMillis() - startTime);
+ resumed = resume(config, testInfo, rescheduler, System.currentTimeMillis() - startTime);
if (!resumed) {
reportFailure(e, listener, config, context, invocationPath);
} else {
@@ -442,14 +444,19 @@
/**
* Attempt to reschedule the failed invocation to resume where it left off.
- * <p/>
- * @see IResumableTest
*
+ * <p>
+ *
+ * @see IResumableTest
* @param config
* @return <code>true</code> if invocation was resumed successfully
*/
- private boolean resume(IConfiguration config, IInvocationContext context,
- IRescheduler rescheduler, long elapsedTime) {
+ private boolean resume(
+ IConfiguration config,
+ TestInformation testInfo,
+ IRescheduler rescheduler,
+ long elapsedTime) {
+ IInvocationContext context = testInfo.getContext();
for (IRemoteTest test : config.getTests()) {
if (test instanceof IResumableTest) {
IResumableTest resumeTest = (IResumableTest)test;
@@ -457,7 +464,7 @@
// resume this config if any test is resumable
IConfiguration resumeConfig = config.clone();
// reuse the same build for the resumed invocation
- ShardBuildCloner.cloneBuildInfos(resumeConfig, resumeConfig, context);
+ ShardBuildCloner.cloneBuildInfos(resumeConfig, resumeConfig, testInfo);
// create a result forwarder, to prevent sending two invocationStarted events
resumeConfig.setTestInvocationListener(new ResumeResultForwarder(
@@ -702,14 +709,26 @@
throws DeviceNotAvailableException, Throwable {
// Create the TestInformation for the invocation
// TODO: Use invocation-id in the workfolder name
- File mWorkFolder = FileUtil.createTempDir("tradefed-invocation-workfolder");
- TestInformation info =
- TestInformation.newBuilder()
- .setInvocationContext(context)
- .setDependenciesFolder(mWorkFolder)
- .build();
- CurrentInvocation.addInvocationInfo(InvocationInfo.WORK_FOLDER, mWorkFolder);
- CleanUpInvocationFiles cleanUpThread = new CleanUpInvocationFiles(info);
+ Object sharedInfoObject =
+ config.getConfigurationObject(ShardHelper.SHARED_TEST_INFORMATION);
+ TestInformation sharedTestInfo = null;
+ TestInformation info = null;
+ if (sharedInfoObject != null) {
+ sharedTestInfo = (TestInformation) sharedInfoObject;
+ // During sharding we share everything except the invocation context
+ info = TestInformation.createModuleTestInfo(sharedTestInfo, context);
+ }
+ if (info == null) {
+ File mWorkFolder = FileUtil.createTempDir("tradefed-invocation-workfolder");
+ info =
+ TestInformation.newBuilder()
+ .setInvocationContext(context)
+ .setDependenciesFolder(mWorkFolder)
+ .build();
+ }
+ CurrentInvocation.addInvocationInfo(InvocationInfo.WORK_FOLDER, info.dependenciesFolder());
+
+ CleanUpInvocationFiles cleanUpThread = new CleanUpInvocationFiles(info, config);
Runtime.getRuntime().addShutdownHook(cleanUpThread);
registerExecutionFiles(info.executionFiles());
@@ -769,6 +788,7 @@
// Seed our TF objects to the Guice scope
scope.seed(IRescheduler.class, rescheduler);
scope.seedConfiguration(config);
+ boolean sharding = false;
try {
ILeveledLogOutput leveledLogOutput = config.getLogOutput();
leveledLogOutput.init();
@@ -852,7 +872,7 @@
}
}
- boolean sharding = invocationPath.shardConfig(config, info, rescheduler, listener);
+ sharding = invocationPath.shardConfig(config, info, rescheduler, listener);
if (sharding) {
CLog.i(
"Invocation for %s has been sharded, rescheduling",
@@ -890,7 +910,6 @@
scope.exit();
// Ensure build infos are always cleaned up at the end of invocation.
invocationPath.cleanUpBuilds(context, config);
-
// ensure we always deregister the logger
for (String deviceName : context.getDeviceConfigNames()) {
if (!(context.getDevice(deviceName).getIDevice() instanceof StubDevice)) {
@@ -905,10 +924,10 @@
config.cleanConfigurationData();
Runtime.getRuntime().removeShutdownHook(cleanUpThread);
- // Delete the invocation work directory at the end
- FileUtil.recursiveDelete(info.dependenciesFolder());
- // Delete all the execution files
- info.executionFiles().clearFiles();
+ if (!sharding) {
+ // If we are the parent shard, we do not delete the test information
+ deleteInvocationFiles(info, config);
+ }
}
}
@@ -1027,6 +1046,24 @@
return testTag;
}
+ /**
+ * Delete the invocation files if this is the last shard for local sharding or if we are not in
+ * a local sharding situation.
+ */
+ private void deleteInvocationFiles(TestInformation testInfo, IConfiguration config) {
+ Object obj = config.getConfigurationObject(ShardHelper.LAST_SHARD_DETECTOR);
+ if (obj != null) {
+ LastShardDetector lastShardDetector = (LastShardDetector) obj;
+ if (!lastShardDetector.isLastShardDone()) {
+ return;
+ }
+ }
+ // Delete the invocation work directory at the end
+ FileUtil.recursiveDelete(testInfo.dependenciesFolder());
+ // Delete all the execution files
+ testInfo.executionFiles().clearFiles();
+ }
+
/** Helper Thread that ensures host_log is reported in case of killed JVM */
private class ReportHostLog extends Thread {
@@ -1049,17 +1086,16 @@
private class CleanUpInvocationFiles extends Thread {
private TestInformation mTestInfo;
+ private IConfiguration mConfig;
- public CleanUpInvocationFiles(TestInformation currentInfo) {
+ public CleanUpInvocationFiles(TestInformation currentInfo, IConfiguration config) {
mTestInfo = currentInfo;
+ mConfig = config;
}
@Override
public void run() {
- // Delete the invocation work directory at the end
- FileUtil.recursiveDelete(mTestInfo.dependenciesFolder());
- // Delete all the execution files
- mTestInfo.executionFiles().clearFiles();
+ deleteInvocationFiles(mTestInfo, mConfig);
}
}
}
diff --git a/src/com/android/tradefed/invoker/shard/LastShardDetector.java b/src/com/android/tradefed/invoker/shard/LastShardDetector.java
new file mode 100644
index 0000000..075fc7c
--- /dev/null
+++ b/src/com/android/tradefed/invoker/shard/LastShardDetector.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tradefed.invoker.shard;
+
+import com.android.tradefed.invoker.ShardMasterResultForwarder;
+import com.android.tradefed.result.ITestInvocationListener;
+
+/**
+ * When running local sharding, sometimes we only want to execute some actions when the last shard
+ * reaches {@link #invocationEnded(long)}. This reporter allows to detect it.
+ *
+ * @see ShardMasterResultForwarder
+ */
+public final class LastShardDetector implements ITestInvocationListener {
+
+ private boolean mLastShardDone = false;
+
+ @Override
+ public void invocationEnded(long elapsedTime) {
+ mLastShardDone = true;
+ }
+
+ /** Returns True if the last shard had called {@link #invocationEnded(long)}. */
+ public boolean isLastShardDone() {
+ return mLastShardDone;
+ }
+}
diff --git a/src/com/android/tradefed/invoker/shard/ShardBuildCloner.java b/src/com/android/tradefed/invoker/shard/ShardBuildCloner.java
index f83f7fb..ccec6c8 100644
--- a/src/com/android/tradefed/invoker/shard/ShardBuildCloner.java
+++ b/src/com/android/tradefed/invoker/shard/ShardBuildCloner.java
@@ -20,6 +20,7 @@
import com.android.tradefed.config.ConfigurationException;
import com.android.tradefed.config.IConfiguration;
import com.android.tradefed.invoker.IInvocationContext;
+import com.android.tradefed.invoker.TestInformation;
import com.android.tradefed.log.LogUtil.CLog;
/**
@@ -33,10 +34,11 @@
*
* @param fromConfig Original configuration
* @param toConfig cloned configuration recreated from the command line.
- * @param context invocation context
+ * @param testInfo The {@link TestInformation} of the parent shard
*/
public static void cloneBuildInfos(
- IConfiguration fromConfig, IConfiguration toConfig, IInvocationContext context) {
+ IConfiguration fromConfig, IConfiguration toConfig, TestInformation testInfo) {
+ IInvocationContext context = testInfo.getContext();
for (String deviceName : context.getDeviceConfigNames()) {
IBuildInfo toBuild = context.getBuildInfo(deviceName).clone();
try {
@@ -52,5 +54,11 @@
CLog.e(e);
}
}
+ try {
+ toConfig.setConfigurationObject(ShardHelper.SHARED_TEST_INFORMATION, testInfo);
+ } catch (ConfigurationException e) {
+ // Should never happen, no action taken
+ CLog.e(e);
+ }
}
}
diff --git a/src/com/android/tradefed/invoker/shard/ShardHelper.java b/src/com/android/tradefed/invoker/shard/ShardHelper.java
index 391be9f..31f704c 100644
--- a/src/com/android/tradefed/invoker/shard/ShardHelper.java
+++ b/src/com/android/tradefed/invoker/shard/ShardHelper.java
@@ -56,6 +56,9 @@
/** Helper class that handles creating the shards and scheduling them for an invocation. */
public class ShardHelper implements IShardHelper {
+ public static final String LAST_SHARD_DETECTOR = "last_shard_detector";
+ public static final String SHARED_TEST_INFORMATION = "shared_test_information";
+
/**
* List of the list configuration obj that should be clone to each shard in order to avoid state
* issues.
@@ -113,8 +116,11 @@
if (shardCount != null) {
expectedShard = Math.min(shardCount, shardableTests.size());
}
+ // Add a tracker so we know in invocation if the last shard is done running.
+ LastShardDetector lastShard = new LastShardDetector();
ShardMasterResultForwarder resultCollector =
- new ShardMasterResultForwarder(buildMasterShardListeners(config), expectedShard);
+ new ShardMasterResultForwarder(
+ buildMasterShardListeners(config, lastShard), expectedShard);
config.getLogSaver().invocationStarted(context);
resultCollector.invocationStarted(context);
@@ -134,6 +140,11 @@
}
for (int i = 0; i < maxShard; i++) {
IConfiguration shardConfig = config.clone();
+ try {
+ shardConfig.setConfigurationObject(LAST_SHARD_DETECTOR, lastShard);
+ } catch (ConfigurationException e) {
+ throw new RuntimeException(e);
+ }
TestsPoolPoller poller =
new TestsPoolPoller(shardableTests, tokenPool, tracker);
shardConfig.setTest(poller);
@@ -150,6 +161,11 @@
for (IRemoteTest testShard : shardableTests) {
CLog.d("Rescheduling sharded config...");
IConfiguration shardConfig = config.clone();
+ try {
+ shardConfig.setConfigurationObject(LAST_SHARD_DETECTOR, lastShard);
+ } catch (ConfigurationException e) {
+ throw new RuntimeException(e);
+ }
if (config.getCommandOptions().shouldUseDynamicSharding()) {
TestsPoolPoller poller =
new TestsPoolPoller(shardableTests, tokenPool, tracker);
@@ -180,7 +196,7 @@
ShardMasterResultForwarder resultCollector,
int index) {
cloneConfigObject(config, shardConfig);
- ShardBuildCloner.cloneBuildInfos(config, shardConfig, testInfo.getContext());
+ ShardBuildCloner.cloneBuildInfos(config, shardConfig, testInfo);
shardConfig.setTestInvocationListeners(
buildShardListeners(resultCollector, config, config.getTestInvocationListeners()));
@@ -301,13 +317,15 @@
* Builds the {@link ITestInvocationListener} listeners that will collect the results from all
* shards. Currently excludes {@link IShardableListener}s.
*/
- private static List<ITestInvocationListener> buildMasterShardListeners(IConfiguration config) {
+ private static List<ITestInvocationListener> buildMasterShardListeners(
+ IConfiguration config, LastShardDetector lastShardDetector) {
List<ITestInvocationListener> newListeners = new ArrayList<ITestInvocationListener>();
for (ITestInvocationListener l : config.getTestInvocationListeners()) {
if (!(l instanceof IShardableListener)) {
newListeners.add(l);
}
}
+ newListeners.add(lastShardDetector);
return newListeners;
}
diff --git a/tests/src/com/android/tradefed/invoker/TestInvocationMultiTest.java b/tests/src/com/android/tradefed/invoker/TestInvocationMultiTest.java
index a3946b3..e94166c 100644
--- a/tests/src/com/android/tradefed/invoker/TestInvocationMultiTest.java
+++ b/tests/src/com/android/tradefed/invoker/TestInvocationMultiTest.java
@@ -80,6 +80,10 @@
mMockConfig = EasyMock.createMock(IConfiguration.class);
EasyMock.expect(mMockConfig.getPostProcessors()).andReturn(mPostProcessors);
EasyMock.expect(mMockConfig.getRetryDecision()).andReturn(new BaseRetryDecision());
+ EasyMock.expect(mMockConfig.getConfigurationObject(ShardHelper.SHARED_TEST_INFORMATION))
+ .andReturn(null);
+ EasyMock.expect(mMockConfig.getConfigurationObject(ShardHelper.LAST_SHARD_DETECTOR))
+ .andReturn(null);
mMockRescheduler = EasyMock.createMock(IRescheduler.class);
mMockTestListener = EasyMock.createMock(ITestInvocationListener.class);
mMockLogSaver = EasyMock.createMock(ILogSaver.class);
diff --git a/tests/src/com/android/tradefed/invoker/TestInvocationTest.java b/tests/src/com/android/tradefed/invoker/TestInvocationTest.java
index 9e009ac..6092441 100644
--- a/tests/src/com/android/tradefed/invoker/TestInvocationTest.java
+++ b/tests/src/com/android/tradefed/invoker/TestInvocationTest.java
@@ -1333,6 +1333,9 @@
*/
@Test
public void testInvoke_shardableTest_legacy() throws Throwable {
+ TestInformation info =
+ TestInformation.newBuilder().setInvocationContext(mStubInvocationMetadata).build();
+ mStubConfiguration.setConfigurationObject(ShardHelper.SHARED_TEST_INFORMATION, info);
String command = "empty --test-tag t";
String[] commandLine = {"empty", "--test-tag", "t"};
int shardCount = 2;
@@ -1386,6 +1389,9 @@
/** Test that the before sharding log is properly carried even with auto-retry. */
@Test
public void testInvoke_shardableTest_autoRetry() throws Throwable {
+ TestInformation info =
+ TestInformation.newBuilder().setInvocationContext(mStubInvocationMetadata).build();
+ mStubConfiguration.setConfigurationObject(ShardHelper.SHARED_TEST_INFORMATION, info);
List<ITestInvocationListener> listenerList =
mStubConfiguration.getTestInvocationListeners();
ILogSaverListener logSaverListener = EasyMock.createMock(ILogSaverListener.class);