src/com/android/tradefed/result/CollectingTestListener.java

/*
 * Copyright (C) 2010 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.android.tradefed.result;

import com.android.ddmlib.testrunner.TestResult.TestStatus;
import com.android.tradefed.build.IBuildInfo;
import com.android.tradefed.config.Option;
import com.android.tradefed.invoker.IInvocationContext;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
import com.android.tradefed.retry.MergeStrategy;

import com.google.common.annotations.VisibleForTesting;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * A {@link ITestInvocationListener} that will collect all test results.
 *
 * <p>Although the data structures used in this object are thread-safe, the {@link
 * ITestInvocationListener} callbacks must be called in the correct order.
 */
public class CollectingTestListener implements ITestInvocationListener, ILogSaverListener {

    @Option(
            name = "aggregate-metrics",
            description = "attempt to add test metrics values for test runs with the same name.")
    private boolean mIsAggregateMetrics = false;

    /** Toggle the 'aggregate metrics' option */
    protected void setIsAggregrateMetrics(boolean aggregate) {
        mIsAggregateMetrics = aggregate;
    }

    private IInvocationContext mContext;
    private IBuildInfo mBuildInfo;
    private Map<String, IInvocationContext> mModuleContextMap = new HashMap<>();
    // Use LinkedHashMap to provide consistent iterations over the keys.
    private Map<String, List<TestRunResult>> mTestRunResultMap =
            Collections.synchronizedMap(new LinkedHashMap<>());

    private IInvocationContext mCurrentModuleContext = null;
    private TestRunResult mCurrentTestRunResult = new TestRunResult();
    /** True if the default initialized mCurrentTestRunResult has its original value. */
    private boolean mDefaultRun = true;
    /** Track whether or not a test run is currently in progress */
    private boolean mRunInProgress = false;

    private Map<String, LogFile> mNonAssociatedLogFiles = new LinkedHashMap<>();

    // Tracks if mStatusCounts are accurate, or if they need to be recalculated
    private AtomicBoolean mIsCountDirty = new AtomicBoolean(true);
    // Tracks if the expected count is accurate, or if it needs to be recalculated.
    private AtomicBoolean mIsExpectedCountDirty = new AtomicBoolean(true);
    private int mExpectedCount = 0;

    // Represents the merged test results. This should not be accessed directly since it's only
    // calculated when needed.
    private final List<TestRunResult> mMergedTestRunResults = new ArrayList<>();
    // Represents the number of tests in each TestStatus state of the merged test results. Indexed
    // by TestStatus.ordinal()
    private int[] mStatusCounts = new int[TestStatus.values().length];

    private MergeStrategy mStrategy = MergeStrategy.ONE_TESTCASE_PASS_IS_PASS;

    /** Sets the {@link MergeStrategy} to use when merging results. */
    public void setMergeStrategy(MergeStrategy strategy) {
        mStrategy = strategy;
    }

    /**
     * Return the primary build info that was reported via {@link
     * #invocationStarted(IInvocationContext)}. Primary build is the build returned by the first
     * build provider of the running configuration. Returns null if there is no context (no build to
     * test case).
     */
    public IBuildInfo getPrimaryBuildInfo() {
        if (mContext == null) {
            return null;
        } else {
            return mContext.getBuildInfos().get(0);
        }
    }

    /**
     * Return the invocation context that was reported via {@link
     * #invocationStarted(IInvocationContext)}
     */
    public IInvocationContext getInvocationContext() {
        return mContext;
    }

    /**
     * Returns the build info.
     *
     * @deprecated rely on the {@link IBuildInfo} from {@link #getInvocationContext()}.
     */
    @Deprecated
    public IBuildInfo getBuildInfo() {
        return mBuildInfo;
    }

    /**
     * Set the build info. Should only be used for testing.
     *
     * @deprecated Not necessary for testing anymore.
     */
    @VisibleForTesting
    @Deprecated
    public void setBuildInfo(IBuildInfo buildInfo) {
        mBuildInfo = buildInfo;
    }

    /** {@inheritDoc} */
    @Override
    public void invocationStarted(IInvocationContext context) {
        mContext = context;
        mBuildInfo = getPrimaryBuildInfo();
    }

    /** {@inheritDoc} */
    @Override
    public void invocationEnded(long elapsedTime) {
        // ignore
    }

    /** {@inheritDoc} */
    @Override
    public void invocationFailed(Throwable cause) {
        // ignore
    }

    /** {@inheritDoc} */
    @Override
    public void invocationFailed(FailureDescription failure) {
        // ignore
    }

    @Override
    public void testModuleStarted(IInvocationContext moduleContext) {
        mCurrentModuleContext = moduleContext;
    }

    @Override
    public void testModuleEnded() {
        mCurrentModuleContext = null;
    }

    /** {@inheritDoc} */
    @Override
    public void testRunStarted(String name, int numTests) {
        testRunStarted(name, numTests, 0);
    }

    private TestRunResult getNewRunResult() {
        TestRunResult result = new TestRunResult();
        if (mDefaultRun) {
            result = mCurrentTestRunResult;
            mDefaultRun = false;
        }
        result.setAggregateMetrics(mIsAggregateMetrics);
        return result;
    }

    /** {@inheritDoc} */
    @Override
    public void testRunStarted(String name, int numTests, int attemptNumber) {
        testRunStarted(name, numTests, attemptNumber, System.currentTimeMillis());
    }

    /** {@inheritDoc} */
    @Override
    public void testRunStarted(String name, int numTests, int attemptNumber, long startTime) {
        setCountDirty();
        // Only testRunStarted can affect the expected count.
        mIsExpectedCountDirty.set(true);

        // Associate the run name with the current module context
        if (mCurrentModuleContext != null) {
            mModuleContextMap.put(name, mCurrentModuleContext);
        }

        // Add the list of maps if the run doesn't exist
        if (!mTestRunResultMap.containsKey(name)) {
            mTestRunResultMap.put(name, new LinkedList<>());
        }
        List<TestRunResult> results = mTestRunResultMap.get(name);

        // Set the current test run result based on the attempt
        if (attemptNumber < results.size()) {
            if (results.get(attemptNumber) == null) {
                throw new RuntimeException(
                        "Test run results should never be null in internal structure.");
            }
        } else if (attemptNumber == results.size()) {
            // new run
            TestRunResult result = getNewRunResult();
            results.add(result);
        } else {
            int size = results.size();
            for (int i = size; i < attemptNumber; i++) {
                TestRunResult result = getNewRunResult();
                result.testRunStarted(name, numTests, startTime);
                String errorMessage =
                        String.format(
                                "Run attempt %s of %s did not exists, but got attempt %s."
                                        + " This is a placeholder for the missing attempt.",
                                i, name, attemptNumber);
                result.testRunFailed(errorMessage);
                result.testRunEnded(0L, new HashMap<String, Metric>());
                results.add(result);
            }
            // New current run
            TestRunResult newResult = getNewRunResult();
            results.add(newResult);
        }
        mCurrentTestRunResult = results.get(attemptNumber);

        mCurrentTestRunResult.testRunStarted(name, numTests, startTime);
        mRunInProgress = true;
    }

    /** {@inheritDoc} */
    @Override
    public void testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics) {
        setCountDirty();
        mCurrentTestRunResult.testRunEnded(elapsedTime, runMetrics);
        mRunInProgress = false;
    }

    /** {@inheritDoc} */
    @Override
    public void testRunFailed(String errorMessage) {
        setCountDirty();
        mCurrentTestRunResult.testRunFailed(errorMessage);
    }

    /** {@inheritDoc} */
    @Override
    public void testRunFailed(FailureDescription failure) {
        setCountDirty();
        mCurrentTestRunResult.testRunFailed(failure);
    }

    /** {@inheritDoc} */
    @Override
    public void testRunStopped(long elapsedTime) {
        setCountDirty();
        mCurrentTestRunResult.testRunStopped(elapsedTime);
    }

    /** {@inheritDoc} */
    @Override
    public void testStarted(TestDescription test) {
        testStarted(test, System.currentTimeMillis());
    }

    /** {@inheritDoc} */
    @Override
    public void testStarted(TestDescription test, long startTime) {
        setCountDirty();
        mCurrentTestRunResult.testStarted(test, startTime);
    }

    /** {@inheritDoc} */
    @Override
    public void testEnded(TestDescription test, HashMap<String, Metric> testMetrics) {
        testEnded(test, System.currentTimeMillis(), testMetrics);
    }

    /** {@inheritDoc} */
    @Override
    public void testEnded(TestDescription test, long endTime, HashMap<String, Metric> testMetrics) {
        setCountDirty();
        mCurrentTestRunResult.testEnded(test, endTime, testMetrics);
    }

    /** {@inheritDoc} */
    @Override
    public void testFailed(TestDescription test, String trace) {
        setCountDirty();
        mCurrentTestRunResult.testFailed(test, trace);
    }

    @Override
    public void testFailed(TestDescription test, FailureDescription failure) {
        setCountDirty();
        mCurrentTestRunResult.testFailed(test, failure);
    }

    @Override
    public void testAssumptionFailure(TestDescription test, String trace) {
        setCountDirty();
        mCurrentTestRunResult.testAssumptionFailure(test, trace);
    }

    @Override
    public void testIgnored(TestDescription test) {
        setCountDirty();
        mCurrentTestRunResult.testIgnored(test);
    }

    /** {@inheritDoc} */
    @Override
    public void logAssociation(String dataName, LogFile logFile) {
        if (mRunInProgress) {
            mCurrentTestRunResult.testLogSaved(dataName, logFile);
        } else {
            mNonAssociatedLogFiles.put(dataName, logFile);
        }
    }

    /**
     * Gets the results for the current test run.
     *
     * <p>Note the results may not be complete. It is recommended to test the value of {@link
     * TestRunResult#isRunComplete()} and/or (@link TestRunResult#isRunFailure()} as appropriate
     * before processing the results.
     *
     * @return the {@link TestRunResult} representing data collected during last test run
     */
    public TestRunResult getCurrentRunResults() {
        return mCurrentTestRunResult;
    }

    /** Returns the total number of complete tests for all runs. */
    public int getNumTotalTests() {
        computeMergedResults();
        int total = 0;
        for (TestStatus s : TestStatus.values()) {
            total += mStatusCounts[s.ordinal()];
        }
        return total;
    }

    /**
     * Returns the number of expected tests count. Could differ from {@link #getNumTotalTests()} if
     * some tests did not run.
     */
    public synchronized int getExpectedTests() {
        // If expected count is not dirty, no need to do anything
        if (!mIsExpectedCountDirty.compareAndSet(true, false)) {
            return mExpectedCount;
        }

        computeMergedResults();
        mExpectedCount = 0;
        for (TestRunResult result : getMergedTestRunResults()) {
            mExpectedCount += result.getExpectedTestCount();
        }
        return mExpectedCount;
    }

    /** Returns the number of tests in given state for this run. */
    public int getNumTestsInState(TestStatus status) {
        computeMergedResults();
        return mStatusCounts[status.ordinal()];
    }

    /** Returns if the invocation had any failed or assumption failed tests. */
    public boolean hasFailedTests() {
        return getNumAllFailedTests() > 0;
    }

    /** Returns the total number of test runs in a failure state */
    public int getNumAllFailedTestRuns() {
        int count = 0;
        for (TestRunResult result : getMergedTestRunResults()) {
            if (result.isRunFailure()) {
                count++;
            }
        }
        return count;
    }

    /**
     * Returns the total number of tests in a failure state (only failed, assumption failures do not
     * count toward it).
     */
    public int getNumAllFailedTests() {
        return getNumTestsInState(TestStatus.FAILURE);
    }

    /**
     * Return the merged collection of results for all runs across different attempts.
     *
     * <p>If there are multiple results, each test run is merged, with the latest test result
     * overwriting test results of previous runs. Test runs are ordered by attempt number.
     *
     * <p>Metrics for the same attempt will be merged based on the preference set by {@code
     * aggregate-metrics}. The final metrics will be the metrics of the last attempt.
     */
    public List<TestRunResult> getMergedTestRunResults() {
        computeMergedResults();
        return new ArrayList<>(mMergedTestRunResults);
    }

    /**
     * Returns the results for all test runs.
     *
     * @deprecated Use {@link #getMergedTestRunResults()}
     */
    @Deprecated
    public Collection<TestRunResult> getRunResults() {
        return getMergedTestRunResults();
    }

    /**
     * Computes and stores the merged results and the total status counts since both operations are
     * expensive.
     */
    private synchronized void computeMergedResults() {
        // If not dirty, nothing to be done
        if (!mIsCountDirty.compareAndSet(true, false)) {
            return;
        }

        mMergedTestRunResults.clear();
        // Merge results
        if (mTestRunResultMap.isEmpty() && mCurrentTestRunResult.isRunFailure()) {
            // In case of early failure that is a bit untracked, still add it to the list to
            // not loose it.
            CLog.e(
                    "Early failure resulting in no testRunStart. Results might be inconsistent:"
                            + "\n%s",
                    mCurrentTestRunResult.getRunFailureMessage());
            mMergedTestRunResults.add(mCurrentTestRunResult);
        } else {
            for (Entry<String, List<TestRunResult>> results : mTestRunResultMap.entrySet()) {
                TestRunResult res = TestRunResult.merge(results.getValue(), mStrategy);
                if (res == null) {
                    // Merge can return null in case of results being empty.
                    CLog.w("No results for %s", results.getKey());
                } else {
                    mMergedTestRunResults.add(res);
                }
            }
        }
        // Reset counts
        for (TestStatus s : TestStatus.values()) {
            mStatusCounts[s.ordinal()] = 0;
        }

        // Calculate results
        for (TestRunResult result : mMergedTestRunResults) {
            for (TestStatus s : TestStatus.values()) {
                mStatusCounts[s.ordinal()] += result.getNumTestsInState(s);
            }
        }
    }

    /**
     * Keep dirty count as AtomicBoolean to ensure when accessed from another thread the state is
     * consistent.
     */
    private void setCountDirty() {
        mIsCountDirty.set(true);
    }

    /**
     * Return all the names for all the test runs.
     *
     * <p>These test runs may have run multiple times with different attempts.
     */
    public Collection<String> getTestRunNames() {
        return new ArrayList<String>(mTestRunResultMap.keySet());
    }

    /**
     * Gets all the attempts for a {@link TestRunResult} of a given test run.
     *
     * @param testRunName The name given by {{@link #testRunStarted(String, int)}.
     * @return All {@link TestRunResult} for a given test run, ordered by attempts.
     */
    public List<TestRunResult> getTestRunAttempts(String testRunName) {
        return mTestRunResultMap.get(testRunName);
    }

    /**
     * Gets all the results for a given attempt.
     *
     * @param attempt The attempt we want results for.
     * @return All {@link TestRunResult} for a given attempt.
     */
    public List<TestRunResult> getTestRunForAttempts(int attempt) {
        List<TestRunResult> allResultForAttempts = new ArrayList<>();
        for (Entry<String, List<TestRunResult>> runInfo : mTestRunResultMap.entrySet()) {
            if (attempt < runInfo.getValue().size()) {
                TestRunResult attemptRes = runInfo.getValue().get(attempt);
                allResultForAttempts.add(attemptRes);
            }
        }
        return allResultForAttempts;
    }

    /**
     * Returns whether a given test run name has any results.
     *
     * @param testRunName The name given by {{@link #testRunStarted(String, int)}.
     */
    public boolean hasTestRunResultsForName(String testRunName) {
        return mTestRunResultMap.containsKey(testRunName);
    }

    /**
     * Returns the number of attempts for a given test run name.
     *
     * @param testRunName The name given by {{@link #testRunStarted(String, int)}.
     */
    public int getTestRunAttemptCount(String testRunName) {
        List<TestRunResult> results = mTestRunResultMap.get(testRunName);
        if (results == null) {
            return 0;
        }
        return results.size();
    }

    /**
     * Return the {@link TestRunResult} for a single attempt.
     *
     * @param testRunName The name given by {{@link #testRunStarted(String, int)}.
     * @param attempt The attempt id.
     * @return The {@link TestRunResult} for the given name and attempt id or {@code null} if it
     *     does not exist.
     */
    public TestRunResult getTestRunAtAttempt(String testRunName, int attempt) {
        List<TestRunResult> results = mTestRunResultMap.get(testRunName);
        if (results == null || attempt < 0 || attempt >= results.size()) {
            return null;
        }

        return results.get(attempt);
    }

    /**
     * Returns the {@link IInvocationContext} of the module associated with the results.
     *
     * @param testRunName The name given by {{@link #testRunStarted(String, int)}.
     * @return The {@link IInvocationContext} of the module for a given test run name {@code null}
     *     if there are no results for that name.
     */
    public IInvocationContext getModuleContextForRunResult(String testRunName) {
        return mModuleContextMap.get(testRunName);
    }

    /** Returns a copy of the map containing all the logged file not associated with a test run. */
    public Map<String, LogFile> getNonAssociatedLogFiles() {
        return new LinkedHashMap<>(mNonAssociatedLogFiles);
    }

    /**
     * Allows to clear the results for a given run name. Should only be used in some cases like the
     * aggregator of results.
     */
    protected final synchronized void clearResultsForName(String testRunName) {
        setCountDirty();
        mTestRunResultMap.remove(testRunName);
    }
}