| /* |
| * Copyright (C) 2010 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.android.cts.tradefed.result; |
| |
| import com.android.cts.tradefed.build.CtsBuildHelper; |
| import com.android.cts.tradefed.device.DeviceInfoCollector; |
| import com.android.cts.tradefed.testtype.CtsTest; |
| import com.android.ddmlib.Log; |
| import com.android.ddmlib.Log.LogLevel; |
| import com.android.ddmlib.testrunner.TestIdentifier; |
| import com.android.tradefed.build.IBuildInfo; |
| import com.android.tradefed.build.IFolderBuildInfo; |
| import com.android.tradefed.config.Option; |
| import com.android.tradefed.log.LogUtil.CLog; |
| import com.android.tradefed.result.ILogFileSaver; |
| import com.android.tradefed.result.ITestInvocationListener; |
| import com.android.tradefed.result.InputStreamSource; |
| import com.android.tradefed.result.LogDataType; |
| import com.android.tradefed.result.LogFileSaver; |
| import com.android.tradefed.result.TestSummary; |
| import com.android.tradefed.util.FileUtil; |
| import com.android.tradefed.util.StreamUtil; |
| |
| import org.kxml2.io.KXmlSerializer; |
| |
| import java.io.File; |
| import java.io.FileNotFoundException; |
| import java.io.FileOutputStream; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.OutputStream; |
| import java.util.Map; |
| import java.util.regex.Matcher; |
| import java.util.regex.Pattern; |
| |
| /** |
| * Writes results to an XML files in the CTS format. |
| * <p/> |
| * Collects all test info in memory, then dumps to file when invocation is complete. |
| * <p/> |
| * Outputs xml in format governed by the cts_result.xsd |
| */ |
| public class CtsXmlResultReporter implements ITestInvocationListener { |
| private static final String LOG_TAG = "CtsXmlResultReporter"; |
| |
| static final String TEST_RESULT_FILE_NAME = "testResult.xml"; |
| private static final String CTS_RESULT_FILE_VERSION = "1.14"; |
| private static final String[] CTS_RESULT_RESOURCES = {"cts_result.xsl", "cts_result.css", |
| "logo.gif", "newrule-green.png"}; |
| |
| /** the XML namespace */ |
| static final String ns = null; |
| |
| static final String RESULT_TAG = "TestResult"; |
| static final String PLAN_ATTR = "testPlan"; |
| static final String STARTTIME_ATTR = "starttime"; |
| |
| private static final String REPORT_DIR_NAME = "output-file-path"; |
| @Option(name=REPORT_DIR_NAME, description="root file system path to directory to store xml " + |
| "test results and associated logs. If not specified, results will be stored at " + |
| "<cts root>/repository/results") |
| protected File mReportDir = null; |
| |
| // listen in on the plan option provided to CtsTest |
| @Option(name = CtsTest.PLAN_OPTION, description = "the test plan to run.") |
| private String mPlanName = "NA"; |
| |
| // listen in on the continue-session option provided to CtsTest |
| @Option(name = CtsTest.CONTINUE_OPTION, description = "the test result session to continue.") |
| private Integer mContinueSessionId = null; |
| |
| @Option(name = "quiet-output", description = "Mute display of test results.") |
| private boolean mQuietOutput = false; |
| |
| @Option(name = "result-server", description = "Server to publish test results.") |
| private String mResultServer; |
| |
| protected IBuildInfo mBuildInfo; |
| private String mStartTime; |
| private String mDeviceSerial; |
| private TestResults mResults = new TestResults(); |
| private TestPackageResult mCurrentPkgResult = null; |
| private boolean mIsDeviceInfoRun = false; |
| private ResultReporter mReporter; |
| private File mLogDir; |
| private String mSuiteName; |
| |
| private static final Pattern mPtsLogPattern = Pattern.compile("(.*)\\+\\+\\+\\+(.*)"); |
| |
| public void setReportDir(File reportDir) { |
| mReportDir = reportDir; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void invocationStarted(IBuildInfo buildInfo) { |
| mBuildInfo = buildInfo; |
| if (!(buildInfo instanceof IFolderBuildInfo)) { |
| throw new IllegalArgumentException("build info is not a IFolderBuildInfo"); |
| } |
| IFolderBuildInfo ctsBuild = (IFolderBuildInfo)buildInfo; |
| CtsBuildHelper ctsBuildHelper = getBuildHelper(ctsBuild); |
| mDeviceSerial = buildInfo.getDeviceSerial() == null ? "unknown_device" : |
| buildInfo.getDeviceSerial(); |
| if (mContinueSessionId != null) { |
| CLog.d("Continuing session %d", mContinueSessionId); |
| // reuse existing directory |
| TestResultRepo resultRepo = new TestResultRepo(ctsBuildHelper.getResultsDir()); |
| mResults = resultRepo.getResult(mContinueSessionId); |
| if (mResults == null) { |
| throw new IllegalArgumentException(String.format("Could not find session %d", |
| mContinueSessionId)); |
| } |
| mPlanName = resultRepo.getSummaries().get(mContinueSessionId).getTestPlan(); |
| mStartTime = resultRepo.getSummaries().get(mContinueSessionId).getStartTime(); |
| mReportDir = resultRepo.getReportDir(mContinueSessionId); |
| } else { |
| if (mReportDir == null) { |
| mReportDir = ctsBuildHelper.getResultsDir(); |
| } |
| // create a unique directory for saving results, using old cts host convention |
| // TODO: in future, consider using LogFileSaver to create build-specific directories |
| mReportDir = new File(mReportDir, TimeUtil.getResultTimestamp()); |
| mReportDir.mkdirs(); |
| mStartTime = getTimestamp(); |
| logResult("Created result dir %s", mReportDir.getName()); |
| } |
| mSuiteName = ctsBuildHelper.getSuiteName(); |
| mReporter = new ResultReporter(mResultServer, mSuiteName); |
| |
| // TODO: allow customization of log dir |
| // create a unique directory for saving logs, with same name as result dir |
| File rootLogDir = getBuildHelper(ctsBuild).getLogsDir(); |
| mLogDir = new File(rootLogDir, mReportDir.getName()); |
| mLogDir.mkdirs(); |
| } |
| |
| /** |
| * Helper method to retrieve the {@link CtsBuildHelper}. |
| * @param ctsBuild |
| */ |
| CtsBuildHelper getBuildHelper(IFolderBuildInfo ctsBuild) { |
| CtsBuildHelper buildHelper = new CtsBuildHelper(ctsBuild.getRootDir()); |
| try { |
| buildHelper.validateStructure(); |
| } catch (FileNotFoundException e) { |
| // just log an error - it might be expected if we failed to retrieve a build |
| CLog.e("Invalid CTS build %s", ctsBuild.getRootDir()); |
| } |
| return buildHelper; |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testLog(String dataName, LogDataType dataType, InputStreamSource dataStream) { |
| try { |
| File logFile = getLogFileSaver().saveAndZipLogData(dataName, dataType, |
| dataStream.createInputStream()); |
| logResult(String.format("Saved log %s", logFile.getName())); |
| } catch (IOException e) { |
| CLog.e("Failed to write log for %s", dataName); |
| } |
| } |
| |
| /** |
| * Return the {@link ILogFileSaver} to use. |
| * <p/> |
| * Exposed for unit testing. |
| */ |
| ILogFileSaver getLogFileSaver() { |
| return new LogFileSaver(mLogDir); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testRunStarted(String name, int numTests) { |
| if (mCurrentPkgResult != null && !name.equals(mCurrentPkgResult.getAppPackageName())) { |
| // display results from previous run |
| logCompleteRun(mCurrentPkgResult); |
| } |
| mIsDeviceInfoRun = name.equals(DeviceInfoCollector.APP_PACKAGE_NAME); |
| if (mIsDeviceInfoRun) { |
| logResult("Collecting device info"); |
| } else { |
| if (mCurrentPkgResult == null || !name.equals(mCurrentPkgResult.getAppPackageName())) { |
| logResult("-----------------------------------------"); |
| logResult("Test package %s started", name); |
| logResult("-----------------------------------------"); |
| } |
| mCurrentPkgResult = mResults.getOrCreatePackage(name); |
| } |
| |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testStarted(TestIdentifier test) { |
| mCurrentPkgResult.insertTest(test); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testFailed(TestFailure status, TestIdentifier test, String trace) { |
| mCurrentPkgResult.reportTestFailure(test, CtsTestStatus.FAIL, trace); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testEnded(TestIdentifier test, Map<String, String> testMetrics) { |
| collectPtsResults(test, testMetrics); |
| mCurrentPkgResult.reportTestEnded(test); |
| Test result = mCurrentPkgResult.findTest(test); |
| String stack = result.getStackTrace() == null ? "" : "\n" + result.getStackTrace(); |
| logResult("%s#%s %s %s", test.getClassName(), test.getTestName(), result.getResult(), |
| stack); |
| } |
| |
| /** |
| * Collect Pts results for both device and host tests to the package result. |
| * @param test test ran |
| * @param testMetrics test metrics which can contain performance result for device tests |
| */ |
| private void collectPtsResults(TestIdentifier test, Map<String, String> testMetrics) { |
| // device test can have performance results in testMetrics |
| String perfResult = PtsReportUtil.getPtsResultFromMetrics(testMetrics); |
| // host test should be checked in PtsHostStore. |
| if (perfResult == null) { |
| perfResult = PtsHostStore.removePtsResult(mDeviceSerial, test); |
| } |
| if (perfResult != null) { |
| // PTS result is passed in Summary++++Details format. |
| // Extract Summary and Details, and pass them. |
| Matcher m = mPtsLogPattern.matcher(perfResult); |
| if (m.find()) { |
| mCurrentPkgResult.reportPerformanceResult(test, CtsTestStatus.PASS, m.group(1), |
| m.group(2)); |
| } else { |
| logResult("PTS Result unrecognizable:" + perfResult); |
| } |
| } |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) { |
| if (mIsDeviceInfoRun) { |
| mResults.populateDeviceInfoMetrics(runMetrics); |
| } else { |
| mCurrentPkgResult.populateMetrics(runMetrics); |
| } |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void invocationEnded(long elapsedTime) { |
| // display the results of the last completed run |
| if (mCurrentPkgResult != null) { |
| logCompleteRun(mCurrentPkgResult); |
| } |
| if (mReportDir == null || mStartTime == null) { |
| // invocationStarted must have failed, abort |
| CLog.w("Unable to create XML report"); |
| return; |
| } |
| |
| File reportFile = getResultFile(mReportDir); |
| createXmlResult(reportFile, mStartTime, elapsedTime); |
| copyFormattingFiles(mReportDir); |
| zipResults(mReportDir); |
| |
| try { |
| mReporter.reportResult(reportFile); |
| } catch (IOException e) { |
| CLog.e(e); |
| } |
| } |
| |
| private void logResult(String format, Object... args) { |
| if (mQuietOutput) { |
| CLog.i(format, args); |
| } else { |
| Log.logAndDisplay(LogLevel.INFO, mDeviceSerial, String.format(format, args)); |
| } |
| } |
| |
| private void logCompleteRun(TestPackageResult pkgResult) { |
| if (pkgResult.getAppPackageName().equals(DeviceInfoCollector.APP_PACKAGE_NAME)) { |
| logResult("Device info collection complete"); |
| return; |
| } |
| logResult("%s package complete: Passed %d, Failed %d, Not Executed %d", |
| pkgResult.getAppPackageName(), pkgResult.countTests(CtsTestStatus.PASS), |
| pkgResult.countTests(CtsTestStatus.FAIL), |
| pkgResult.countTests(CtsTestStatus.NOT_EXECUTED)); |
| } |
| |
| /** |
| * Creates a report file and populates it with the report data from the completed tests. |
| */ |
| private void createXmlResult(File reportFile, String startTimestamp, long elapsedTime) { |
| String endTime = getTimestamp(); |
| OutputStream stream = null; |
| try { |
| stream = createOutputResultStream(reportFile); |
| KXmlSerializer serializer = new KXmlSerializer(); |
| serializer.setOutput(stream, "UTF-8"); |
| serializer.startDocument("UTF-8", false); |
| serializer.setFeature( |
| "http://xmlpull.org/v1/doc/features.html#indent-output", true); |
| serializer.processingInstruction("xml-stylesheet type=\"text/xsl\" " + |
| "href=\"cts_result.xsl\""); |
| serializeResultsDoc(serializer, startTimestamp, endTime); |
| serializer.endDocument(); |
| String msg = String.format("XML test result file generated at %s. Passed %d, " + |
| "Failed %d, Not Executed %d", mReportDir.getName(), |
| mResults.countTests(CtsTestStatus.PASS), |
| mResults.countTests(CtsTestStatus.FAIL), |
| mResults.countTests(CtsTestStatus.NOT_EXECUTED)); |
| logResult(msg); |
| logResult("Time: %s", TimeUtil.formatElapsedTime(elapsedTime)); |
| } catch (IOException e) { |
| Log.e(LOG_TAG, "Failed to generate report data"); |
| } finally { |
| StreamUtil.closeStream(stream); |
| } |
| } |
| |
| /** |
| * Output the results XML. |
| * |
| * @param serializer the {@link KXmlSerializer} to use |
| * @param startTime the user-friendly starting time of the test invocation |
| * @param endTime the user-friendly ending time of the test invocation |
| * @throws IOException |
| */ |
| private void serializeResultsDoc(KXmlSerializer serializer, String startTime, String endTime) |
| throws IOException { |
| serializer.startTag(ns, RESULT_TAG); |
| serializer.attribute(ns, PLAN_ATTR, mPlanName); |
| serializer.attribute(ns, STARTTIME_ATTR, startTime); |
| serializer.attribute(ns, "endtime", endTime); |
| serializer.attribute(ns, "version", CTS_RESULT_FILE_VERSION); |
| serializer.attribute(ns, "suite", mSuiteName); |
| |
| mResults.serialize(serializer); |
| // TODO: not sure why, but the serializer doesn't like this statement |
| //serializer.endTag(ns, RESULT_TAG); |
| } |
| |
| private File getResultFile(File reportDir) { |
| return new File(reportDir, TEST_RESULT_FILE_NAME); |
| } |
| |
| /** |
| * Creates the output stream to use for test results. Exposed for mocking. |
| */ |
| OutputStream createOutputResultStream(File reportFile) throws IOException { |
| logResult("Created xml report file at file://%s", reportFile.getAbsolutePath()); |
| return new FileOutputStream(reportFile); |
| } |
| |
| /** |
| * Copy the xml formatting files stored in this jar to the results directory |
| * |
| * @param resultsDir |
| */ |
| private void copyFormattingFiles(File resultsDir) { |
| for (String resultFileName : CTS_RESULT_RESOURCES) { |
| InputStream configStream = getClass().getResourceAsStream(String.format("/report/%s", |
| resultFileName)); |
| if (configStream != null) { |
| File resultFile = new File(resultsDir, resultFileName); |
| try { |
| FileUtil.writeToFile(configStream, resultFile); |
| } catch (IOException e) { |
| Log.w(LOG_TAG, String.format("Failed to write %s to file", resultFileName)); |
| } |
| } else { |
| Log.w(LOG_TAG, String.format("Failed to load %s from jar", resultFileName)); |
| } |
| } |
| } |
| |
| /** |
| * Zip the contents of the given results directory. |
| * |
| * @param resultsDir |
| */ |
| private void zipResults(File resultsDir) { |
| try { |
| // create a file in parent directory, with same name as resultsDir |
| File zipResultFile = new File(resultsDir.getParent(), String.format("%s.zip", |
| resultsDir.getName())); |
| FileUtil.createZip(resultsDir, zipResultFile); |
| } catch (IOException e) { |
| Log.w(LOG_TAG, String.format("Failed to create zip for %s", resultsDir.getName())); |
| } |
| } |
| |
| /** |
| * Get a String version of the current time. |
| * <p/> |
| * Exposed so unit tests can mock. |
| */ |
| String getTimestamp() { |
| return TimeUtil.getTimestamp(); |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testRunFailed(String errorMessage) { |
| // ignore |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void testRunStopped(long elapsedTime) { |
| // ignore |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public void invocationFailed(Throwable cause) { |
| // ignore |
| } |
| |
| /** |
| * {@inheritDoc} |
| */ |
| @Override |
| public TestSummary getSummary() { |
| return null; |
| } |
| } |