src/com/android/media/tests/CameraPerformanceTest.java

/*
 * Copyright (C) 2015 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.media.tests;

import com.google.common.collect.ImmutableMultimap;

import com.android.ddmlib.testrunner.TestIdentifier;
import com.android.tradefed.config.OptionClass;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.ITestInvocationListener;

import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;
import org.xmlpull.v1.XmlPullParserFactory;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * This test invocation runs android.hardware.camera2.cts.PerformanceTest -
 * Camera2 API use case performance KPIs, such as camera open time, session creation time,
 * shutter lag etc. The KPI data will be parsed and reported to dashboard.
 */
@OptionClass(alias = "camera-framework")
public class CameraPerformanceTest extends CameraTestBase {

    public CameraPerformanceTest() {
        // Set up the default test info. But this is subject to be overwritten by options passed
        // from commands.
        setTestPackage("android.camera.cts");
        setTestClass("android.hardware.camera2.cts.PerformanceTest");
        setTestRunner("android.support.test.runner.AndroidJUnitRunner");
        setRuKey("camera_framework_performance");
        setTestTimeoutMs(10 * 60 * 1000); // 10 mins
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
        runInstrumentationTest(listener, new CollectingListener(listener));
    }

    /**
     * A listener to collect the output from test run and fatal errors
     */
    private class CollectingListener extends DefaultCollectingListener {

        public CollectingListener(ITestInvocationListener listener) {
            super(listener);
        }

        @Override
        public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) {
            // Pass the test name for a key in the aggregated metrics, because
            // it is used to generate the key of the final metrics to post at the end of test run.
            for (Map.Entry<String, String> metric : testMetrics.entrySet()) {
                getAggregatedMetrics().put(test.getTestName(), metric.getValue());
            }
        }

        @Override
        public void handleTestRunEnded(ITestInvocationListener listener, long elapsedTime,
                Map<String, String> runMetrics) {
            // Report metrics at the end of test run.
            Map<String, String> result = parseResult(getAggregatedMetrics());
            listener.testRunEnded(getTestDurationMs(), result);
        }
    }

    /**
     * Parse Camera Performance KPIs result from the stdout generated by each test run.
     * Then put them all together to post the final report
     *
     * @return a {@link HashMap} that contains pairs of kpiName and kpiValue
     */
    private Map<String, String> parseResult(Map<String, String> metrics) {
        Map<String, String> resultsAll = new HashMap<String, String>();

        CtsResultParserBase parser;
        for (Map.Entry<String, String> metric : metrics.entrySet()) {
            String testMethod = metric.getKey();
            String testResult = metric.getValue();
            CLog.d("test name %s", testMethod);
            CLog.d("test result %s", testResult);
            // Probe which result parser should be used.
            if (shouldUseCtsXmlResultParser(testResult)) {
                parser = new CtsXmlResultParser();
            } else {
                parser = new CtsDelimitedResultParser();
            }

            // Get pairs of { KPI name, KPI value } from stdout that each test outputs.
            // Assuming that a device has both the front and back cameras, parser will return
            // 2 KPIs in HashMap. For an example of testCameraLaunch,
            //   {
            //     ("Camera 0 Camera launch time", "379.2"),
            //     ("Camera 1 Camera launch time", "272.8"),
            //   }
            Map<String, String> testKpis = parser.parse(testResult, testMethod);
            for (String k : testKpis.keySet()) {
                if (resultsAll.containsKey(k)) {
                    throw new RuntimeException(String.format("KPI name (%s) conflicts with " +
                            "the existing names. ", k));
                }
            }
            parser.clear();

            // Put each result together to post the final result
            resultsAll.putAll(testKpis);
        }
        return resultsAll;
    }

    public boolean shouldUseCtsXmlResultParser(String result) {
        final String XML_DECLARATION = "<?xml";
        return (result.startsWith(XML_DECLARATION) ||
                result.startsWith(XML_DECLARATION.toUpperCase()));
    }

    /**
     * Data class of CTS test results for Camera framework performance test
     */
    public static class CtsMetric {
        String testMethod;  // "testSingleCapture"
        String source;      // "android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327"
                            // or "testSingleCapture" (just test method name)
        String message;     // "Camera 0: Camera capture latency"
        String type;        // "lower_better"
        String unit;        // "ms"
        String value;       // "691.0" (is an average of 736.0 688.0 679.0 667.0 686.0)
        String schemaKey;   // RU schema key = message (+ testMethodName if needed), derived

        // eg. "android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327"
        public static final Pattern SOURCE_REGEX = Pattern.compile(
                "^(?<package>[a-zA-Z\\d\\._$]+)#(?<method>[a-zA-Z\\d_$]+)(:\\d+)?");
        // eg. "Camera 0: Camera capture latency"
        public static final Pattern MESSAGE_REGEX = Pattern.compile(
                "^Camera\\s+(?<cameraId>\\d+):\\s+(?<kpiName>.*)");

        CtsMetric(String testMethod, String source, String message, String type, String unit,
                String value) {
            this.testMethod = testMethod;
            this.source = source;
            this.message = message;
            this.type = type;
            this.unit = unit;
            this.value = value;
            this.schemaKey = getRuSchemaKeyName(message);
        }

        public boolean matches(String testMethod, String kpiName) {
            return (this.testMethod.equals(testMethod) && this.message.endsWith(kpiName));
        }

        public String getRuSchemaKeyName(String message) {
            // Note 1: The key shouldn't contain ":" for side by side report.
            String schemaKey = message.replace(":", "");
            // Note 2: Two tests testReprocessingLatency & testReprocessingThroughput have the
            // same metric names to report results. To make the report key name distinct,
            // the test name is added as prefix for these tests for them.
            final String[] TEST_NAMES_AS_PREFIX = {"testReprocessingLatency",
                    "testReprocessingThroughput"};
            for (String testName : TEST_NAMES_AS_PREFIX) {
                if (testMethod.endsWith(testName)) {
                    schemaKey = String.format("%s_%s", testName, schemaKey);
                    break;
                }
            }
            return schemaKey;
        }

        public String getTestMethodNameInSource(String source) {
            Matcher m = SOURCE_REGEX.matcher(source);
            if (!m.matches()) {
                return source;
            }
            return m.group("method");
        }
    }

    /**
     * Base class of CTS test result parser. This is inherited to two derived parsers,
     * {@link CtsDelimitedResultParser} for legacy delimiter separated format and
     * {@link CtsXmlResultParser} for XML typed format introduced since NYC.
     */
    public abstract class CtsResultParserBase {
        // KPIs to be reported. The key is test methods and the value is KPIs in the method.
        private ImmutableMultimap<String, String> mReportingKpis =
                new ImmutableMultimap.Builder<String, String>()
                        .put("testCameraLaunch", "Camera launch time")
                        .put("testCameraLaunch", "Camera start preview time")
                        .put("testSingleCapture", "Camera capture result latency")
                        .put("testReprocessingLatency", "YUV reprocessing shot to shot latency")
                        .put("testReprocessingLatency", "opaque reprocessing shot to shot latency")
                        .put("testReprocessingThroughput", "YUV reprocessing capture latency")
                        .put("testReprocessingThroughput", "opaque reprocessing capture latency")
                        .build();

        protected CtsMetric mSummary;
        protected List<CtsMetric> mDetails = new ArrayList<>();

        /**
         * Parse Camera Performance KPIs result first, then leave the only KPIs that matter.
         *
         * @param result String to be parsed
         * @param testMethod test method name used to leave the only metric that matters
         * @return a {@link HashMap} that contains kpiName and kpiValue
         */
        public abstract Map<String, String> parse(String result, String testMethod);

        protected Map<String, String> filter(List<CtsMetric> metrics, String testMethod) {
            Map<String, String> filtered = new HashMap<String, String>();
            for (CtsMetric metric : metrics) {
                for (String kpiName : mReportingKpis.get(testMethod)) {
                    // Post the data only when it matches with the given methods and KPI names.
                    if (metric.matches(testMethod, kpiName)) {
                        filtered.put(metric.schemaKey, metric.value);
                    }
                }
            }
            return filtered;
        }

        protected void setSummary(CtsMetric summary) {
            mSummary = summary;
        }

        protected void addDetail(CtsMetric detail) {
            mDetails.add(detail);
        }

        protected List<CtsMetric> getDetails() {
            return mDetails;
        }

        void clear() {
            mSummary = null;
            mDetails.clear();
        }

        public <E extends Number> Double getAverage(List<E> list) {
            double sum = 0;
            int size = list.size();
            for (E num : list) {
                sum += num.doubleValue();
            }
            if (size == 0) {
                return Double.NaN;
            }
            return (sum / size);
        }
    }

    /**
     * Parses the stdout generated by the underlying instrumentation test
     * and returns it to test runner for later reporting.
     *
     * Format:
     *   (summary message)| |(type)|(unit)|(value) ++++
     *   (source)|(message)|(type)|(unit)|(value)... +++
     *   ...
     *
     * Example:
     *   Camera launch average time for Camera 1| |lower_better|ms|586.6++++
     *   android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171|
     *       Camera 0: Camera open time|lower_better|ms|74.0 100.0 70.0 67.0 82.0 +++
     *   android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171|
     *       Camera 0: Camera configure stream time|lower_better|ms|9.0 5.0 5.0 8.0 5.0
     *   ...
     *
     * See also com.android.cts.util.ReportLog for the format detail.
     *
     */
    public class CtsDelimitedResultParser extends CtsResultParserBase {
        private static final String LOG_SEPARATOR = "\\+\\+\\+";
        private static final String SUMMARY_SEPARATOR = "\\+\\+\\+\\+";
        private Pattern mSummaryRegex =
                Pattern.compile(
                        "^(?<message>[^|]+)\\| \\|(?<type>[^|]+)\\|(?<unit>[^|]+)\\|(?<value>[0-9 .]+)");
        private Pattern mDetailRegex =
                Pattern.compile(
                        "^(?<source>[^|]+)\\|(?<message>[^|]+)\\|(?<type>[^|]+)\\|(?<unit>[^|]+)\\|"
                                + "(?<values>[0-9 .]+)");

        @Override
        public Map<String, String> parse(String result, String testMethod) {
            parseToCtsMetrics(result, testMethod);
            return filter(getDetails(), testMethod);
        }

        void parseToCtsMetrics(String result, String testMethod) {
            // Split summary and KPIs from stdout passes as parameter.
            String[] output = result.split(SUMMARY_SEPARATOR);
            if (output.length != 2) {
                throw new RuntimeException("Value not in the correct format");
            }
            Matcher summaryMatcher = mSummaryRegex.matcher(output[0].trim());

            // Parse summary.
            // Example: "Camera launch average time for Camera 1| |lower_better|ms|586.6++++"
            if (summaryMatcher.matches()) {
                setSummary(new CtsMetric(testMethod,
                        null,
                        summaryMatcher.group("message"),
                        summaryMatcher.group("type"),
                        summaryMatcher.group("unit"),
                        summaryMatcher.group("value")));
            } else {
                // Fall through since the summary is not posted as results.
                CLog.w("Summary not in the correct format");
            }

            // Parse KPIs.
            // Example: "android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171|Camera 0: Camera open time|lower_better|ms|74.0 100.0 70.0 67.0 82.0 +++"
            String[] details = output[1].split(LOG_SEPARATOR);
            for (String detail : details) {
                Matcher detailMatcher = mDetailRegex.matcher(detail.trim());
                if (detailMatcher.matches()) {
                    // get average of kpi values
                    List<Double> values = new ArrayList<>();
                    for (String value : detailMatcher.group("values").split("\\s+")) {
                        values.add(Double.parseDouble(value));
                    }
                    String kpiValue = String.format("%.1f", getAverage(values));
                    addDetail(new CtsMetric(
                            testMethod,
                            detailMatcher.group("source"),
                            detailMatcher.group("message"),
                            detailMatcher.group("type"),
                            detailMatcher.group("unit"),
                            kpiValue));
                } else {
                    throw new RuntimeException("KPI not in the correct format");
                }
            }
        }
    }

    /**
     * Parses the CTS test results in a XML format introduced since NYC.
     *
     * Format:
     *   <Summary>
     *       <Metric source="android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327"
     *               message="Camera capture result average latency for all cameras "
     *               score_type="lower_better"
     *               score_unit="ms"
     *           <Value>353.9</Value>
     *       </Metric>
     *   </Summary>
     *   <Detail>
     *       <Metric source="android.hardware.camera2.cts.PerformanceTest#testSingleCapture:303"
     *               message="Camera 0: Camera capture latency"
     *               score_type="lower_better"
     *               score_unit="ms">
     *           <Value>335.0</Value>
     *           <Value>302.0</Value>
     *           <Value>316.0</Value>
     *       </Metric>
     *   </Detail>
     *  }
     * See also com.android.compatibility.common.util.ReportLog for the format detail.
     */
    public class CtsXmlResultParser extends CtsResultParserBase {
        private static final String ENCODING = "UTF-8";
        // XML constants
        private static final String DETAIL_TAG = "Detail";
        private static final String METRIC_TAG = "Metric";
        private static final String MESSAGE_ATTR = "message";
        private static final String SCORETYPE_ATTR = "score_type";
        private static final String SCOREUNIT_ATTR = "score_unit";
        private static final String SOURCE_ATTR = "source";
        private static final String SUMMARY_TAG = "Summary";
        private static final String VALUE_TAG = "Value";
        private String mTestMethod;

        @Override
        public Map<String, String> parse(String result, String testMethod) {
            try {
                mTestMethod = testMethod;
                XmlPullParserFactory factory = XmlPullParserFactory.newInstance();
                XmlPullParser parser = factory.newPullParser();
                parser.setInput(new ByteArrayInputStream(result.getBytes(ENCODING)), ENCODING);
                parser.nextTag();
                parse(parser);
                return filter(getDetails(), testMethod);
            } catch (XmlPullParserException | IOException e) {
                throw new RuntimeException("Failed to parse results in XML.", e);
            }
        }

        /**
         * Parses a {@link CtsMetric} from the given XML parser.
         * @param parser
         * @throws IOException
         * @throws XmlPullParserException
         */
        private void parse(XmlPullParser parser) throws XmlPullParserException, IOException {
            parser.require(XmlPullParser.START_TAG, null, SUMMARY_TAG);
            parser.nextTag();
            setSummary(parseToCtsMetrics(parser));
            parser.nextTag();
            parser.require(XmlPullParser.END_TAG, null, SUMMARY_TAG);
            parser.nextTag();
            if (parser.getName().equals(DETAIL_TAG)) {
                while (parser.nextTag() == XmlPullParser.START_TAG) {
                    addDetail(parseToCtsMetrics(parser));
                }
                parser.require(XmlPullParser.END_TAG, null, DETAIL_TAG);
            }
        }

        CtsMetric parseToCtsMetrics(XmlPullParser parser)
                throws IOException, XmlPullParserException {
            parser.require(XmlPullParser.START_TAG, null, METRIC_TAG);
            String source = parser.getAttributeValue(null, SOURCE_ATTR);
            String message = parser.getAttributeValue(null, MESSAGE_ATTR);
            String type = parser.getAttributeValue(null, SCORETYPE_ATTR);
            String unit = parser.getAttributeValue(null, SCOREUNIT_ATTR);
            List<Double> values = new ArrayList<>();
            while (parser.nextTag() == XmlPullParser.START_TAG) {
                parser.require(XmlPullParser.START_TAG, null, VALUE_TAG);
                values.add(Double.parseDouble(parser.nextText()));
                parser.require(XmlPullParser.END_TAG, null, VALUE_TAG);
            }
            String kpiValue = String.format("%.1f", getAverage(values));
            parser.require(XmlPullParser.END_TAG, null, METRIC_TAG);
            return new CtsMetric(mTestMethod, source, message, type, unit, kpiValue);
        }
    }
}