Revert "Revert "Merge prod-tests/src/com/android/media/tests/ from platform/tools/tradefederation to src/com/android/media/tests/ BUG:63819116""
This reverts commit 14672dddc6362f1ae0857ee5255ef95628cb98db.
Change-Id: Ie52f8d188281ff1351b40c309c6b19ae160ded4e
diff --git a/src/com/android/media/tests/AdbScreenrecordTest.java b/src/com/android/media/tests/AdbScreenrecordTest.java
new file mode 100644
index 0000000..eb50f50
--- /dev/null
+++ b/src/com/android/media/tests/AdbScreenrecordTest.java
@@ -0,0 +1,433 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.media.tests;
+
+import com.android.ddmlib.CollectingOutputReceiver;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.IFileEntry;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.CommandResult;
+import com.android.tradefed.util.CommandStatus;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.RunUtil;
+
+import java.io.File;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Tests adb command "screenrecord", i.e. "adb screenrecord [--size] [--bit-rate] [--time-limit]"
+ *
+ * <p>The test use the above command to record a video of DUT's screen. It then tries to verify that
+ * a video was actually recorded and that the video is a valid video file. It currently uses
+ * 'avprobe' to do the video analysis along with extracting parameters from the adb command's
+ * output.
+ */
+public class AdbScreenrecordTest implements IDeviceTest, IRemoteTest {
+
+ //===================================================================
+ // TEST OPTIONS
+ //===================================================================
+ @Option(name = "run-key", description = "Run key for the test")
+ private String mRunKey = "AdbScreenRecord";
+
+ @Option(name = "time-limit", description = "Recording time in seconds", isTimeVal = true)
+ private long mRecordTimeInSeconds = -1;
+
+ @Option(name = "size", description = "Video Size: 'widthxheight', e.g. '1280x720'")
+ private String mVideoSize = null;
+
+ @Option(name = "bit-rate", description = "Video bit rate in megabits per second, e.g. 4000000")
+ private long mBitRate = -1;
+
+ //===================================================================
+ // CLASS VARIABLES
+ //===================================================================
+ private ITestDevice mDevice;
+ private TestRunHelper mTestRunHelper;
+
+ //===================================================================
+ // CONSTANTS
+ //===================================================================
+ private static final long TEST_TIMEOUT_MS = 5 * 60 * 1000; // 5 min
+ private static final long DEVICE_SYNC_MS = 5 * 60 * 1000; // 5 min
+ private static final long POLLING_INTERVAL_MS = 5 * 1000; // 5 sec
+ private static final long CMD_TIMEOUT_MS = 5 * 1000; // 5 sec
+ private static final String ERR_OPTION_MALFORMED = "Test option %1$s is not correct [%2$s]";
+ private static final String OPTION_TIME_LIMIT = "--time-limit";
+ private static final String OPTION_SIZE = "--size";
+ private static final String OPTION_BITRATE = "--bit-rate";
+ private static final String RESULT_KEY_RECORDED_FRAMES = "recorded_frames";
+ private static final String RESULT_KEY_RECORDED_LENGTH = "recorded_length";
+ private static final String RESULT_KEY_VERIFIED_DURATION = "verified_duration";
+ private static final String RESULT_KEY_VERIFIED_BITRATE = "verified_bitrate";
+ private static final String TEST_FILE = "/sdcard/screenrecord_test.mp4";
+ private static final String AVPROBE_NOT_INSTALLED =
+ "Program 'avprobe' is not installed on host '%1$s'";
+ private static final String REGEX_IS_VIDEO_OK =
+ "Duration: (\\d\\d:\\d\\d:\\d\\d.\\d\\d).+bitrate: (\\d+ .b\\/s)";
+ private static final String AVPROBE_STR = "avprobe";
+
+ //===================================================================
+ // ENUMS
+ //===================================================================
+ enum HOST_SOFTWARE {
+ AVPROBE
+ }
+
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ @Override
+ public ITestDevice getDevice() {
+ return mDevice;
+ }
+
+ /** Main test function invoked by test harness */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ initializeTest(listener);
+
+ CLog.i("Verify required software is installed on host");
+ verifyRequiredSoftwareIsInstalled(HOST_SOFTWARE.AVPROBE);
+
+ mTestRunHelper.startTest(1);
+
+ Map<String, String> resultsDictionary = new HashMap<String, String>();
+ try {
+ CLog.i("Verify that test options are valid");
+ if (!verifyTestParameters()) {
+ return;
+ }
+
+ // "resultDictionary" can be used to post results to dashboards like BlackBox
+ resultsDictionary = runTest(resultsDictionary, TEST_TIMEOUT_MS);
+ } finally {
+ final String metricsStr = Arrays.toString(resultsDictionary.entrySet().toArray());
+ CLog.i("Uploading metrics values:\n" + metricsStr);
+ mTestRunHelper.endTest(resultsDictionary);
+ }
+ }
+
+ /**
+ * Test code that calls "adb screenrecord" and checks for pass/fail criterias
+ *
+ * <p>
+ *
+ * <ul>
+ * <li>1. Run adb screenrecord command
+ * <li>2. Wait until there is a video file; fail if none appears
+ * <li>3. Analyze adb output and extract recorded number of frames and video length
+ * <li>4. Pull recorded video file off device
+ * <li>5. Using avprobe, analyze video file and extract duration and bitrate
+ * <li>6. Return extracted results
+ * </ul>
+ *
+ * @throws DeviceNotAvailableException
+ */
+ private Map<String, String> runTest(Map<String, String> results, final long timeout)
+ throws DeviceNotAvailableException {
+ final CollectingOutputReceiver receiver = new CollectingOutputReceiver();
+ final String cmd = generateAdbScreenRecordCommand();
+ final String deviceFileName = getAbsoluteFilename();
+
+ CLog.i("START Execute device shell command: '" + cmd + "'");
+ getDevice().executeShellCommand(cmd, receiver, timeout, TimeUnit.MILLISECONDS, 3);
+ String adbOutput = receiver.getOutput();
+ CLog.i(adbOutput);
+ CLog.i("END Execute device shell command");
+
+ CLog.i("Wait for recorded file: " + deviceFileName);
+ if (!waitForFile(getDevice(), timeout, deviceFileName)) {
+ mTestRunHelper.reportFailure("Recorded test file not found");
+ // Since we don't have a file, no need to delete it; we can return here
+ return results;
+ }
+
+ CLog.i("Get number of recorded frames and recorded length from adb output");
+ if (!extractVideoDataFromAdbOutput(adbOutput, results)) {
+ deleteFileFromDevice(deviceFileName);
+ return results;
+ }
+
+ CLog.i("Get duration and bitrate info from video file using '" + AVPROBE_STR + "'");
+ try {
+ extractDurationAndBitrateFromVideoFileUsingAvprobe(deviceFileName, results);
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ deleteFileFromDevice(deviceFileName);
+ return results;
+ }
+
+ /** Convert a string on form HH:mm:ss.SS to nearest number of seconds */
+ private long convertBitrateToKilobits(String bitrate) {
+ Matcher m = Pattern.compile("(\\d+) (.)b\\/s").matcher(bitrate);
+ if (!m.matches()) {
+ return -1;
+ }
+
+ final String unit = m.group(2).toUpperCase();
+ long factor = 1;
+ switch (unit) {
+ case "K":
+ factor = 1;
+ break;
+ case "M":
+ factor = 1000;
+ break;
+ case "G":
+ factor = 1000000;
+ break;
+ }
+
+ long rate = Long.parseLong(m.group(1));
+
+ return rate * factor;
+ }
+
+ /**
+ * Convert a string on form HH:mm:ss.SS to nearest number of seconds
+ *
+ * @throws ParseException
+ */
+ private long convertDurationToMilliseconds(String duration) throws ParseException {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SS");
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+ Date convertedDate = sdf.parse("1970-01-01 " + duration);
+ return convertedDate.getTime();
+ }
+
+ /**
+ * Deletes a file off a device
+ *
+ * @param deviceFileName - path and filename to file to be deleted
+ * @throws DeviceNotAvailableException
+ */
+ private void deleteFileFromDevice(String deviceFileName) throws DeviceNotAvailableException {
+ CLog.i("Delete file from device: " + deviceFileName);
+ getDevice().executeShellCommand("rm -f " + deviceFileName);
+ }
+
+ /**
+ * Extracts duration and bitrate data from a video file
+ *
+ * @throws DeviceNotAvailableException
+ * @throws ParseException
+ */
+ private boolean extractDurationAndBitrateFromVideoFileUsingAvprobe(
+ String deviceFileName, Map<String, String> results)
+ throws DeviceNotAvailableException, ParseException {
+ CLog.i("Check if the recorded file has some data in it: " + deviceFileName);
+ IFileEntry video = getDevice().getFileEntry(deviceFileName);
+ if (video == null || video.getFileEntry().getSizeValue() < 1) {
+ mTestRunHelper.reportFailure("Video Entry info failed");
+ return false;
+ }
+
+ final File recordedVideo = getDevice().pullFile(deviceFileName);
+ CLog.i("Recorded video file: " + recordedVideo.getAbsolutePath());
+
+ CommandResult result =
+ RunUtil.getDefault()
+ .runTimedCmd(
+ CMD_TIMEOUT_MS,
+ AVPROBE_STR,
+ "-loglevel",
+ "info",
+ recordedVideo.getAbsolutePath());
+
+ // Remove file from host machine
+ FileUtil.deleteFile(recordedVideo);
+
+ if (result.getStatus() != CommandStatus.SUCCESS) {
+ mTestRunHelper.reportFailure(AVPROBE_STR + " command failed");
+ return false;
+ }
+
+ String data = result.getStderr();
+ CLog.i("data: " + data);
+ if (data == null || data.isEmpty()) {
+ mTestRunHelper.reportFailure(AVPROBE_STR + " output data is empty");
+ return false;
+ }
+
+ Matcher m = Pattern.compile(REGEX_IS_VIDEO_OK).matcher(data);
+ if (!m.find()) {
+ final String errMsg =
+ "Video verification failed; no matching verification pattern found";
+ mTestRunHelper.reportFailure(errMsg);
+ return false;
+ }
+
+ String duration = m.group(1);
+ long durationInMilliseconds = convertDurationToMilliseconds(duration);
+ String bitrate = m.group(2);
+ long bitrateInKilobits = convertBitrateToKilobits(bitrate);
+
+ results.put(RESULT_KEY_VERIFIED_DURATION, Long.toString(durationInMilliseconds / 1000));
+ results.put(RESULT_KEY_VERIFIED_BITRATE, Long.toString(bitrateInKilobits));
+ return true;
+ }
+
+ /** Extracts recorded number of frames and recorded video length from adb output */
+ private boolean extractVideoDataFromAdbOutput(String adbOutput, Map<String, String> results) {
+ final String regEx = "recorded (\\d+) frames in (\\d+) second";
+ Matcher m = Pattern.compile(regEx).matcher(adbOutput);
+ if (!m.find()) {
+ mTestRunHelper.reportFailure("Regular Expression did not find recorded frames");
+ return false;
+ }
+
+ int recordedFrames = Integer.parseInt(m.group(1));
+ int recordedLength = Integer.parseInt(m.group(2));
+ CLog.i("Recorded frames: " + recordedFrames);
+ CLog.i("Recorded length: " + recordedLength);
+ if (recordedFrames <= 0) {
+ mTestRunHelper.reportFailure("No recorded frames detected");
+ return false;
+ }
+
+ results.put(RESULT_KEY_RECORDED_FRAMES, Integer.toString(recordedFrames));
+ results.put(RESULT_KEY_RECORDED_LENGTH, Integer.toString(recordedLength));
+ return true;
+ }
+
+ /** Generates an adb command from passed in test options */
+ private String generateAdbScreenRecordCommand() {
+ final String SPACE = " ";
+ StringBuilder sb = new StringBuilder(128);
+ sb.append("screenrecord --verbose ").append(getAbsoluteFilename());
+
+ // Add test options if they have been passed in to the test
+ if (mRecordTimeInSeconds != -1) {
+ final long timeLimit = TimeUnit.MILLISECONDS.toSeconds(mRecordTimeInSeconds);
+ sb.append(SPACE).append(OPTION_TIME_LIMIT).append(SPACE).append(timeLimit);
+ }
+
+ if (mVideoSize != null) {
+ sb.append(SPACE).append(OPTION_SIZE).append(SPACE).append(mVideoSize);
+ }
+
+ if (mBitRate != -1) {
+ sb.append(SPACE).append(OPTION_BITRATE).append(SPACE).append(mBitRate);
+ }
+
+ return sb.toString();
+ }
+
+ /** Returns absolute path to device recorded video file */
+ private String getAbsoluteFilename() {
+ return TEST_FILE;
+ }
+
+ /** Performs test initialization steps */
+ private void initializeTest(ITestInvocationListener listener)
+ throws UnsupportedOperationException, DeviceNotAvailableException {
+ TestIdentifier testId = new TestIdentifier(getClass().getCanonicalName(), mRunKey);
+
+ // Allocate helpers
+ mTestRunHelper = new TestRunHelper(listener, testId);
+
+ getDevice().disableKeyguard();
+ getDevice().waitForDeviceAvailable(DEVICE_SYNC_MS);
+
+ CLog.i("Sync device time to host time");
+ getDevice().setDate(new Date());
+ }
+
+ /** Verifies that required software is installed on host machine */
+ private void verifyRequiredSoftwareIsInstalled(HOST_SOFTWARE software) {
+ String swName = "";
+ switch (software) {
+ case AVPROBE:
+ swName = AVPROBE_STR;
+ CommandResult result =
+ RunUtil.getDefault().runTimedCmd(CMD_TIMEOUT_MS, swName, "-version");
+ String output = result.getStdout();
+ if (result.getStatus() == CommandStatus.SUCCESS && output.startsWith(swName)) {
+ return;
+ }
+ break;
+ }
+
+ CLog.i("Program '" + swName + "' not found, report test failure");
+ String hostname = RunUtil.getDefault().runTimedCmd(CMD_TIMEOUT_MS, "hostname").getStdout();
+
+ String err = String.format(AVPROBE_NOT_INSTALLED, (hostname == null) ? "" : hostname);
+ throw new RuntimeException(err);
+ }
+
+ /** Verifies that passed in test parameters are legitimate */
+ private boolean verifyTestParameters() {
+ if (mRecordTimeInSeconds != -1 && mRecordTimeInSeconds < 1) {
+ final String error =
+ String.format(ERR_OPTION_MALFORMED, OPTION_TIME_LIMIT, mRecordTimeInSeconds);
+ mTestRunHelper.reportFailure(error);
+ return false;
+ }
+
+ if (mVideoSize != null) {
+ final String videoSizeRegEx = "\\d+x\\d+";
+ Matcher m = Pattern.compile(videoSizeRegEx).matcher(mVideoSize);
+ if (!m.matches()) {
+ final String error = String.format(ERR_OPTION_MALFORMED, OPTION_SIZE, mVideoSize);
+ mTestRunHelper.reportFailure(error);
+ return false;
+ }
+ }
+
+ if (mBitRate != -1 && mBitRate < 1) {
+ final String error = String.format(ERR_OPTION_MALFORMED, OPTION_BITRATE, mBitRate);
+ mTestRunHelper.reportFailure(error);
+ return false;
+ }
+
+ return true;
+ }
+
+ /** Checks for existence of a file on the device */
+ private static boolean waitForFile(
+ ITestDevice device, final long timeout, final String absoluteFilename)
+ throws DeviceNotAvailableException {
+ final long checkFileStartTime = System.currentTimeMillis();
+
+ do {
+ RunUtil.getDefault().sleep(POLLING_INTERVAL_MS);
+ if (device.doesFileExist(absoluteFilename)) {
+ return true;
+ }
+ } while (System.currentTimeMillis() - checkFileStartTime < timeout);
+
+ return false;
+ }
+}
diff --git a/src/com/android/media/tests/AudioJitterTest.java b/src/com/android/media/tests/AudioJitterTest.java
new file mode 100644
index 0000000..7058cc7
--- /dev/null
+++ b/src/com/android/media/tests/AudioJitterTest.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.CollectingOutputReceiver;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A harness that launches AudioJitter tool and reports result.
+ */
+public class AudioJitterTest implements IDeviceTest, IRemoteTest {
+
+ private static final String RUN_KEY = "audiojitter";
+ private static final long TIMEOUT_MS = 5 * 60 * 1000; // 5 min
+ private static final int MAX_ATTEMPTS = 3;
+ private static final Map<String, String> METRICS_KEY_MAP = createMetricsKeyMap();
+
+ private ITestDevice mDevice;
+
+ private static final String DEVICE_TEMPORARY_DIR_PATH = "/data/local/tmp/";
+ private static final String JITTER_BINARY_FILENAME = "sljitter";
+ private static final String JITTER_BINARY_DEVICE_PATH =
+ DEVICE_TEMPORARY_DIR_PATH + JITTER_BINARY_FILENAME;
+
+ private static Map<String, String> createMetricsKeyMap() {
+ Map<String, String> result = new HashMap<String, String>();
+ result.put("min_jitter_ticks", "min_jitter_ticks");
+ result.put("min_jitter_ms", "min_jitter_ms");
+ result.put("min_jitter_period_id", "min_jitter_period_id");
+ result.put("max_jitter_ticks", "max_jitter_ticks");
+ result.put("max_jitter_ms", "max_jitter_ms");
+ result.put("max_jitter_period_id", "max_jitter_period_id");
+ result.put("mark_jitter_ticks", "mark_jitter_ticks");
+ result.put("mark_jitter_ms", "mark_jitter_ms");
+ result.put("max_cb_done_delay_ms", "max_cb_done_delay_ms");
+ result.put("max_thread_delay_ms", "max_thread_delay_ms");
+ result.put("max_render_delay_ms", "max_render_delay_ms");
+ result.put("drift_rate", "drift_rate");
+ result.put("error_ms", "error_ms");
+ result.put("min_error_ms", "min_error_ms");
+ result.put("max_error_ms", "max_error_ms");
+ return Collections.unmodifiableMap(result);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ITestDevice getDevice() {
+ return mDevice;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ TestIdentifier testId = new TestIdentifier(getClass().getCanonicalName(), RUN_KEY);
+ ITestDevice device = getDevice();
+
+ listener.testRunStarted(RUN_KEY, 0);
+ listener.testStarted(testId);
+
+ long testStartTime = System.currentTimeMillis();
+ Map<String, String> metrics = new HashMap<String, String>();
+ String errMsg = null;
+
+ // start jitter and wait for process to complete
+ CollectingOutputReceiver receiver = new CollectingOutputReceiver();
+ device.executeShellCommand(JITTER_BINARY_DEVICE_PATH, receiver,
+ TIMEOUT_MS, TimeUnit.MILLISECONDS, MAX_ATTEMPTS);
+ String resultStr = receiver.getOutput();
+
+ if (resultStr != null) {
+ // parse result
+ CLog.i("== Jitter result ==");
+ Map<String, String> jitterResult = parseResult(resultStr);
+ if (jitterResult == null) {
+ errMsg = "Failed to parse Jitter result.";
+ } else {
+ metrics = jitterResult;
+ }
+ } else {
+ errMsg = "Jitter result not found.";
+ }
+
+ if (errMsg != null) {
+ CLog.e(errMsg);
+ listener.testFailed(testId, errMsg);
+ listener.testEnded(testId, metrics);
+ listener.testRunFailed(errMsg);
+ } else {
+ long durationMs = System.currentTimeMillis() - testStartTime;
+ listener.testEnded(testId, metrics);
+ listener.testRunEnded(durationMs, metrics);
+ }
+ }
+
+ /**
+ * Parse Jitter result.
+ *
+ * @param result Jitter result output
+ * @return a {@link HashMap} that contains metrics keys and results
+ */
+ private Map<String, String> parseResult(String result) {
+ Map<String, String> resultMap = new HashMap<String, String>();
+ String lines[] = result.split("\\r?\\n");
+ for (String line: lines) {
+ line = line.trim().replaceAll(" +", " ");
+ String[] tokens = line.split(" ");
+ if (tokens.length >= 2) {
+ String metricName = tokens[0];
+ String metricValue = tokens[1];
+ if (METRICS_KEY_MAP.containsKey(metricName)) {
+ CLog.i(String.format("%s: %s", metricName, metricValue));
+ resultMap.put(METRICS_KEY_MAP.get(metricName), metricValue);
+ }
+ }
+ }
+ return resultMap;
+ }
+}
diff --git a/src/com/android/media/tests/AudioLevelUtility.java b/src/com/android/media/tests/AudioLevelUtility.java
new file mode 100644
index 0000000..3aee1fb
--- /dev/null
+++ b/src/com/android/media/tests/AudioLevelUtility.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.media.tests;
+
+import com.android.ddmlib.CollectingOutputReceiver;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+
+import java.util.concurrent.TimeUnit;
+
+/** Class to provide audio level utility functions for a test device */
+public class AudioLevelUtility {
+
+ public static int extractDeviceAudioLevelFromAdbShell(ITestDevice device)
+ throws DeviceNotAvailableException {
+
+ final String ADB_SHELL_DUMPSYS_AUDIO = "dumpsys audio";
+ final String STREAM_MUSIC = "- STREAM_MUSIC:";
+ final String HEADSET = "(headset): ";
+
+ final CollectingOutputReceiver receiver = new CollectingOutputReceiver();
+
+ device.executeShellCommand(
+ ADB_SHELL_DUMPSYS_AUDIO, receiver, 300, TimeUnit.MILLISECONDS, 1);
+ final String shellOutput = receiver.getOutput();
+ if (shellOutput == null || shellOutput.isEmpty()) {
+ return -1;
+ }
+
+ int audioLevel = -1;
+ int pos = shellOutput.indexOf(STREAM_MUSIC);
+ if (pos != -1) {
+ pos = shellOutput.indexOf(HEADSET, pos);
+ if (pos != -1) {
+ final int start = pos + HEADSET.length();
+ final int stop = shellOutput.indexOf(",", start);
+ if (stop != -1) {
+ final String audioLevelStr = shellOutput.substring(start, stop);
+ try {
+ audioLevel = Integer.parseInt(audioLevelStr);
+ } catch (final NumberFormatException e) {
+ CLog.e(e.getMessage());
+ audioLevel = 1;
+ }
+ }
+ }
+ }
+
+ return audioLevel;
+ }
+}
diff --git a/src/com/android/media/tests/AudioLoopbackImageAnalyzer.java b/src/com/android/media/tests/AudioLoopbackImageAnalyzer.java
new file mode 100644
index 0000000..3b6d767
--- /dev/null
+++ b/src/com/android/media/tests/AudioLoopbackImageAnalyzer.java
@@ -0,0 +1,475 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.media.tests;
+
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.util.Pair;
+
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import javax.imageio.ImageIO;
+
+/**
+ * Class that analyzes a screenshot captured from AudioLoopback test. There is a wave form in the
+ * screenshot that has specific colors (TARGET_COLOR). This class extracts those colors and analyzes
+ * wave amplitude, duration and form and make a decision if it's a legitimate wave form or not.
+ */
+public class AudioLoopbackImageAnalyzer {
+
+ // General
+ private static final int HORIZONTAL_THRESHOLD = 10;
+ private static final int VERTICAL_THRESHOLD = 0;
+ private static final int PRIMARY_WAVE_COLOR = 0xFF1E4A99;
+ private static final int SECONDARY_WAVE_COLOR = 0xFF1D4998;
+ private static final int[] TARGET_COLORS_TABLET =
+ new int[] {PRIMARY_WAVE_COLOR, SECONDARY_WAVE_COLOR};
+ private static final int[] TARGET_COLORS_PHONE = new int[] {PRIMARY_WAVE_COLOR};
+
+ private static final float EXPERIMENTAL_WAVE_MAX_TABLET = 69.0f; // In percent of image height
+ private static final float EXPERIMENTAL_WAVE_MAX_PHONE = 32.0f; // In percent of image height
+
+ // Image
+ private static final int TABLET_SCREEN_MIN_WIDTH = 1700;
+ private static final int TABLET_SCREEN_MIN_HEIGHT = 2300;
+
+ // Duration parameters
+ // Max duration should not span more than 2 of the 11 sections in the graph
+ // Min duration should not be less than 1/4 of a section
+ private static final float SECTION_WIDTH_IN_PERCENT = 100 * 1 / 11; // In percent of image width
+ private static final float DURATION_MIN = SECTION_WIDTH_IN_PERCENT / 4;
+
+ // Amplitude
+ // Required numbers of column for a response
+ private static final int MIN_NUMBER_OF_COLUMNS = 4;
+ // The difference between two amplitude columns should not be more than this
+ private static final float MAX_ALLOWED_COLUMN_DECREASE = 0.50f;
+ // Only check MAX_ALLOWED_COLUMN_DECREASE up to this number
+ private static final float MIN_NUMBER_OF_DECREASING_COLUMNS = 8;
+
+ enum Result {
+ PASS,
+ FAIL,
+ UNKNOWN
+ }
+
+ private static class Amplitude {
+ public int maxHeight = -1;
+ public int zeroCounter = 0;
+ }
+
+ public static Pair<Result, String> analyzeImage(String imgFile) {
+ final String FN_TAG = "AudioLoopbackImageAnalyzer.analyzeImage";
+
+ BufferedImage img = null;
+ try {
+ final File f = new File(imgFile);
+ img = ImageIO.read(f);
+ } catch (final IOException e) {
+ CLog.e(e);
+ throw new RuntimeException("Error loading image file '" + imgFile + "'");
+ }
+
+ final int width = img.getWidth();
+ final int height = img.getHeight();
+
+ CLog.i("image width=" + width + ", height=" + height);
+
+ // Compute thresholds and min/max values based on image witdh, height
+ final float waveMax;
+ final int[] targetColors;
+ final int amplitudeCenterMaxDiff;
+ final float maxDuration;
+ final int minNrOfZeroesBetweenAmplitudes;
+
+ if (width >= TABLET_SCREEN_MIN_WIDTH && height >= TABLET_SCREEN_MIN_HEIGHT) {
+ CLog.i("Apply TABLET config values");
+ waveMax = EXPERIMENTAL_WAVE_MAX_TABLET;
+ amplitudeCenterMaxDiff = 40;
+ minNrOfZeroesBetweenAmplitudes = 8;
+ maxDuration = 3 * SECTION_WIDTH_IN_PERCENT;
+ targetColors = TARGET_COLORS_TABLET;
+ } else {
+ waveMax = EXPERIMENTAL_WAVE_MAX_PHONE;
+ amplitudeCenterMaxDiff = 20;
+ minNrOfZeroesBetweenAmplitudes = 5;
+ maxDuration = 2.5f * SECTION_WIDTH_IN_PERCENT;
+ targetColors = TARGET_COLORS_PHONE;
+ }
+
+ // Amplitude
+ // Max height should be about 80% of wave max.
+ // Min height should be about 40% of wave max.
+ final float AMPLITUDE_MAX_VALUE = waveMax * 0.8f;
+ final float AMPLITUDE_MIN_VALUE = waveMax * 0.4f;
+
+ final int[] vertical = new int[height];
+ final int[] horizontal = new int[width];
+
+ projectPixelsToXAxis(img, targetColors, horizontal, width, height);
+ filter(horizontal, HORIZONTAL_THRESHOLD);
+ final Pair<Integer, Integer> durationBounds = getBounds(horizontal);
+ if (!boundsWithinRange(durationBounds, 0, width)) {
+ final String fmt = "%1$s Upper/Lower bound along horizontal axis not found";
+ final String err = String.format(fmt, FN_TAG);
+ CLog.w(err);
+ return new Pair<Result, String>(Result.FAIL, err);
+ }
+
+ projectPixelsToYAxis(img, targetColors, vertical, height, durationBounds);
+ filter(vertical, VERTICAL_THRESHOLD);
+ final Pair<Integer, Integer> amplitudeBounds = getBounds(vertical);
+ if (!boundsWithinRange(durationBounds, 0, height)) {
+ final String fmt = "%1$s: Upper/Lower bound along vertical axis not found";
+ final String err = String.format(fmt, FN_TAG);
+ CLog.w(err);
+ return new Pair<Result, String>(Result.FAIL, err);
+ }
+
+ final int durationLeft = durationBounds.first.intValue();
+ final int durationRight = durationBounds.second.intValue();
+ final int amplitudeTop = amplitudeBounds.first.intValue();
+ final int amplitudeBottom = amplitudeBounds.second.intValue();
+
+ final float amplitude = (amplitudeBottom - amplitudeTop) * 100.0f / height;
+ final float duration = (durationRight - durationLeft) * 100.0f / width;
+
+ CLog.i("AudioLoopbackImageAnalyzer: Amplitude=" + amplitude + ", Duration=" + duration);
+
+ Pair<Result, String> amplResult =
+ analyzeAmplitude(
+ vertical,
+ amplitude,
+ amplitudeTop,
+ amplitudeBottom,
+ AMPLITUDE_MIN_VALUE,
+ AMPLITUDE_MAX_VALUE,
+ amplitudeCenterMaxDiff);
+ if (amplResult.first != Result.PASS) {
+ return amplResult;
+ }
+
+ amplResult =
+ analyzeDuration(
+ horizontal,
+ duration,
+ durationLeft,
+ durationRight,
+ DURATION_MIN,
+ maxDuration,
+ MIN_NUMBER_OF_COLUMNS,
+ minNrOfZeroesBetweenAmplitudes);
+ if (amplResult.first != Result.PASS) {
+ return amplResult;
+ }
+
+ return new Pair<Result, String>(Result.PASS, "");
+ }
+
+ /**
+ * Function to analyze the waveforms duration (how wide it stretches along x-axis) and to make
+ * sure the waveform degrades nicely, i.e. the amplitude columns becomes smaller and smaller
+ * over time.
+ *
+ * @param horizontal - int array with waveforms amplitude values
+ * @param duration - calculated length of duration in percent of screen width
+ * @param durationLeft - index for "horizontal" where waveform starts
+ * @param durationRight - index for "horizontal" where waveform ends
+ * @param durationMin - if duration is below this value, return FAIL and failure reason
+ * @param durationMax - if duration exceed this value, return FAIL and failure reason
+ * @param minNumberOfAmplitudes - min number of amplitudes (columns) in waveform to pass test
+ * @param minNrOfZeroesBetweenAmplitudes - min number of required zeroes between amplitudes
+ * @return - returns result status and failure reason, if any
+ */
+ private static Pair<Result, String> analyzeDuration(
+ int[] horizontal,
+ float duration,
+ int durationLeft,
+ int durationRight,
+ final float durationMin,
+ final float durationMax,
+ final int minNumberOfAmplitudes,
+ final int minNrOfZeroesBetweenAmplitudes) {
+ // This is the tricky one; basically, there should be "columns" that starts
+ // at "durationLeft", with the tallest column to the left and then column
+ // height will drop until it fades completely after "durationRight".
+ final String FN_TAG = "AudioLoopbackImageAnalyzer.analyzeDuration";
+
+ if (duration < durationMin || duration > durationMax) {
+ final String fmt = "%1$s: Duration outside range, value=%2$f, range=(%3$f,%4$f)";
+ return handleError(fmt, FN_TAG, duration, durationMin, durationMax);
+ }
+
+ final ArrayList<Amplitude> amplitudes = new ArrayList<Amplitude>();
+ Amplitude currentAmplitude = null;
+ int zeroCounter = 0;
+
+ for (int i = durationLeft; i < durationRight; i++) {
+ final int v = horizontal[i];
+ if (v == 0) {
+ zeroCounter++;
+ } else {
+ CLog.i("index=" + i + ", v=" + v);
+
+ if (zeroCounter > minNrOfZeroesBetweenAmplitudes) {
+ // Found a new amplitude; update old amplitude
+ // with the "gap" count - i.e. nr of zeroes between the amplitudes
+ if (currentAmplitude != null) {
+ currentAmplitude.zeroCounter = zeroCounter;
+ }
+
+ // Create new Amplitude object
+ currentAmplitude = new Amplitude();
+ amplitudes.add(currentAmplitude);
+ }
+
+ // Reset counter
+ zeroCounter = 0;
+
+ if (currentAmplitude != null && v > currentAmplitude.maxHeight) {
+ currentAmplitude.maxHeight = horizontal[i];
+ }
+ }
+ }
+
+ StringBuilder sb = new StringBuilder(128);
+ int counter = 0;
+ for (final Amplitude a : amplitudes) {
+ CLog.i(
+ sb.append("Amplitude=")
+ .append(counter)
+ .append(", MaxHeight=")
+ .append(a.maxHeight)
+ .append(", ZeroesToNextColumn=")
+ .append(a.zeroCounter)
+ .toString());
+ counter++;
+ sb.setLength(0);
+ }
+
+ if (amplitudes.size() < minNumberOfAmplitudes) {
+ final String fmt = "%1$s: Not enough amplitude columns, value=%2$d";
+ return handleError(fmt, FN_TAG, amplitudes.size());
+ }
+
+ int currentColumnHeight = -1;
+ int oldColumnHeight = -1;
+ for (int i = 0; i < amplitudes.size(); i++) {
+ if (i == 0) {
+ oldColumnHeight = amplitudes.get(i).maxHeight;
+ continue;
+ }
+
+ currentColumnHeight = amplitudes.get(i).maxHeight;
+ if (oldColumnHeight > currentColumnHeight) {
+ // We want at least a good number of columns that declines nicely.
+ // After MIN_NUMBER_OF_DECREASING_COLUMNS, we don't really care that much
+ if (i < MIN_NUMBER_OF_DECREASING_COLUMNS
+ && currentColumnHeight < (oldColumnHeight * MAX_ALLOWED_COLUMN_DECREASE)) {
+ final String fmt =
+ "%1$s: Amplitude column heights declined too much, "
+ + "old=%2$d, new=%3$d, column=%4$d";
+ return handleError(fmt, FN_TAG, oldColumnHeight, currentColumnHeight, i);
+ }
+ oldColumnHeight = currentColumnHeight;
+ } else if (oldColumnHeight == currentColumnHeight) {
+ if (i < MIN_NUMBER_OF_DECREASING_COLUMNS) {
+ final String fmt =
+ "%1$s: Amplitude column heights are same, "
+ + "old=%2$d, new=%3$d, column=%4$d";
+ return handleError(fmt, FN_TAG, oldColumnHeight, currentColumnHeight, i);
+ }
+ } else {
+ final String fmt =
+ "%1$s: Amplitude column heights don't decline, "
+ + "old=%2$d, new=%3$d, column=%4$d";
+ return handleError(fmt, FN_TAG, oldColumnHeight, currentColumnHeight, i);
+ }
+ }
+
+ return new Pair<Result, String>(Result.PASS, "");
+ }
+
+ /**
+ * Function to analyze the waveforms duration (how wide it stretches along x-axis) and to make
+ * sure the waveform degrades nicely, i.e. the amplitude columns becomes smaller and smaller
+ * over time.
+ *
+ * @param vertical - integer array with waveforms amplitude accumulated values
+ * @param amplitude - calculated height of amplitude in percent of screen height
+ * @param amplitudeTop - index in "vertical" array where waveform starts
+ * @param amplitudeBottom - index in "vertical" array where waveform ends
+ * @param amplitudeMin - if amplitude is below this value, return FAIL and failure reason
+ * @param amplitudeMax - if amplitude exceed this value, return FAIL and failure reason
+ * @param amplitudeCenterDiffThreshold - threshold to check that waveform is centered
+ * @return - returns result status and failure reason, if any
+ */
+ private static Pair<Result, String> analyzeAmplitude(
+ int[] vertical,
+ float amplitude,
+ int amplitudeTop,
+ int amplitudeBottom,
+ final float amplitudeMin,
+ final float amplitudeMax,
+ final int amplitudeCenterDiffThreshold) {
+ final String FN_TAG = "AudioLoopbackImageAnalyzer.analyzeAmplitude";
+
+ if (amplitude < amplitudeMin || amplitude > amplitudeMax) {
+ final String fmt = "%1$s: Amplitude outside range, value=%2$f, range=(%3$f,%4$f)";
+ final String err = String.format(fmt, FN_TAG, amplitude, amplitudeMin, amplitudeMax);
+ CLog.w(err);
+ return new Pair<Result, String>(Result.FAIL, err);
+ }
+
+ // Are the amplitude top/bottom centered around the centerline?
+ final int amplitudeCenter = getAmplitudeCenter(vertical, amplitudeTop, amplitudeBottom);
+ final int topDiff = amplitudeCenter - amplitudeTop;
+ final int bottomDiff = amplitudeBottom - amplitudeCenter;
+ final int diff = Math.abs(topDiff - bottomDiff);
+
+ if (diff < amplitudeCenterDiffThreshold) {
+ return new Pair<Result, String>(Result.PASS, "");
+ }
+
+ final String fmt =
+ "%1$s: Amplitude not centered topDiff=%2$d, bottomDiff=%3$d, "
+ + "center=%4$d, diff=%5$d";
+ final String err = String.format(fmt, FN_TAG, topDiff, bottomDiff, amplitudeCenter, diff);
+ CLog.w(err);
+ return new Pair<Result, String>(Result.FAIL, err);
+ }
+
+ private static int getAmplitudeCenter(int[] vertical, int amplitudeTop, int amplitudeBottom) {
+ int max = -1;
+ int center = -1;
+ for (int i = amplitudeTop; i < amplitudeBottom; i++) {
+ if (vertical[i] > max) {
+ max = vertical[i];
+ center = i;
+ }
+ }
+
+ return center;
+ }
+
+ private static void projectPixelsToXAxis(
+ BufferedImage img,
+ final int[] targetColors,
+ int[] horizontal,
+ final int width,
+ final int height) {
+ // "Flatten image" by projecting target colors horizontally,
+ // counting number of found pixels in each column
+ for (int y = 0; y < height; y++) {
+ for (int x = 0; x < width; x++) {
+ final int color = img.getRGB(x, y);
+ for (final int targetColor : targetColors) {
+ if (color == targetColor) {
+ horizontal[x]++;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ private static void projectPixelsToYAxis(
+ BufferedImage img,
+ final int[] targetColors,
+ int[] vertical,
+ int height,
+ Pair<Integer, Integer> horizontalMinMax) {
+
+ final int min = horizontalMinMax.first.intValue();
+ final int max = horizontalMinMax.second.intValue();
+
+ // "Flatten image" by projecting target colors (between min/max) vertically,
+ // counting number of found pixels in each row
+
+ // Pass over y-axis, restricted to horizontalMin, horizontalMax
+ for (int y = 0; y < height; y++) {
+ for (int x = min; x <= max; x++) {
+ final int color = img.getRGB(x, y);
+ for (final int targetColor : targetColors) {
+ if (color == targetColor) {
+ vertical[y]++;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ private static Pair<Integer, Integer> getBounds(int[] array) {
+ // Determine min, max
+ int min = -1;
+ for (int i = 0; i < array.length; i++) {
+ if (array[i] > 0) {
+ min = i;
+ break;
+ }
+ }
+
+ int max = -1;
+ for (int i = array.length - 1; i >= 0; i--) {
+ if (array[i] > 0) {
+ max = i;
+ break;
+ }
+ }
+
+ return new Pair<Integer, Integer>(Integer.valueOf(min), Integer.valueOf(max));
+ }
+
+ private static void filter(int[] array, final int threshold) {
+ // Filter horizontal array; set all values < threshold to 0
+ for (int i = 0; i < array.length; i++) {
+ final int v = array[i];
+ if (v != 0 && v <= threshold) {
+ array[i] = 0;
+ }
+ }
+ }
+
+ private static boolean boundsWithinRange(Pair<Integer, Integer> bounds, int low, int high) {
+ return low <= bounds.first.intValue()
+ && bounds.first.intValue() < high
+ && low <= bounds.second.intValue()
+ && bounds.second.intValue() < high;
+ }
+
+ private static Pair<Result, String> handleError(String fmt, String tag, int arg1) {
+ final String err = String.format(fmt, tag, arg1);
+ CLog.w(err);
+ return new Pair<Result, String>(Result.FAIL, err);
+ }
+
+ private static Pair<Result, String> handleError(
+ String fmt, String tag, int arg1, int arg2, int arg3) {
+ final String err = String.format(fmt, tag, arg1, arg2, arg3);
+ CLog.w(err);
+ return new Pair<Result, String>(Result.FAIL, err);
+ }
+
+ private static Pair<Result, String> handleError(
+ String fmt, String tag, float arg1, float arg2, float arg3) {
+ final String err = String.format(fmt, tag, arg1, arg2, arg3);
+ CLog.w(err);
+ return new Pair<Result, String>(Result.FAIL, err);
+ }
+}
diff --git a/src/com/android/media/tests/AudioLoopbackTest.java b/src/com/android/media/tests/AudioLoopbackTest.java
new file mode 100644
index 0000000..1d9bab5
--- /dev/null
+++ b/src/com/android/media/tests/AudioLoopbackTest.java
@@ -0,0 +1,749 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.media.tests;
+
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+
+import com.android.ddmlib.NullOutputReceiver;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.media.tests.AudioLoopbackTestHelper.LogFileType;
+import com.android.media.tests.AudioLoopbackTestHelper.ResultData;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.RunUtil;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Runs Audio Latency and Audio Glitch test and reports result.
+ *
+ * <p>Strategy for Audio Latency Stress test: RUN test 1000 times. In each iteration, collect result
+ * files from device, parse and collect data in a ResultData object that also keeps track of
+ * location to test files for a particular iteration.
+ *
+ * <p>ANALYZE test results to produce statistics for 1. Latency and Confidence (Min, Max, Mean,
+ * Median) 2. Create CSV file with test run data 3. Print bad test data to host log file 4. Get
+ * number of test runs with valid data to send to dashboard 5. Produce histogram in host log file;
+ * count number of test results that fall into 1 ms wide buckets.
+ *
+ * <p>UPLOAD test results + log files from “bad” runs; i.e. runs that is missing some or all result
+ * data.
+ */
+public class AudioLoopbackTest implements IDeviceTest, IRemoteTest {
+
+ //===================================================================
+ // TEST OPTIONS
+ //===================================================================
+ @Option(name = "run-key", description = "Run key for the test")
+ private String mRunKey = "AudioLoopback";
+
+ @Option(name = "sampling-freq", description = "Sampling Frequency for Loopback app")
+ private String mSamplingFreq = "48000";
+
+ @Option(name = "mic-source", description = "Mic Source for Loopback app")
+ private String mMicSource = "3";
+
+ @Option(name = "audio-thread", description = "Audio Thread for Loopback app")
+ private String mAudioThread = "1";
+
+ @Option(
+ name = "audio-level",
+ description =
+ "Audio Level for Loopback app. A device specific"
+ + "param which makes waveform in loopback test hit 60% to 80% range"
+ )
+ private String mAudioLevel = "-1";
+
+ @Option(name = "test-type", description = "Test type to be executed")
+ private String mTestType = TESTTYPE_LATENCY_STR;
+
+ @Option(name = "buffer-test-duration", description = "Buffer test duration in seconds")
+ private String mBufferTestDuration = "10";
+
+ @Option(name = "key-prefix", description = "Key Prefix for reporting")
+ private String mKeyPrefix = "48000_Mic3_";
+
+ @Option(name = "iterations", description = "Number of test iterations")
+ private int mIterations = 1;
+
+ @Option(name = "baseline_latency", description = "")
+ private float mBaselineLatency = 0f;
+
+ //===================================================================
+ // CLASS VARIABLES
+ //===================================================================
+ private static final Map<String, String> METRICS_KEY_MAP = createMetricsKeyMap();
+ private Map<LogFileType, LogFileData> mFileDataKeyMap;
+ private ITestDevice mDevice;
+ private TestRunHelper mTestRunHelper;
+ private AudioLoopbackTestHelper mLoopbackTestHelper;
+
+ //===================================================================
+ // CONSTANTS
+ //===================================================================
+ private static final String TESTTYPE_LATENCY_STR = "222";
+ private static final String TESTTYPE_GLITCH_STR = "223";
+ private static final long TIMEOUT_MS = 5 * 60 * 1000; // 5 min
+ private static final long DEVICE_SYNC_MS = 5 * 60 * 1000; // 5 min
+ private static final long POLLING_INTERVAL_MS = 5 * 1000;
+ private static final int MAX_ATTEMPTS = 3;
+ private static final int MAX_NR_OF_LOG_UPLOADS = 100;
+
+ private static final int LATENCY_ITERATIONS_LOWER_BOUND = 1;
+ private static final int LATENCY_ITERATIONS_UPPER_BOUND = 10000;
+ private static final int GLITCH_ITERATIONS_LOWER_BOUND = 1;
+ private static final int GLITCH_ITERATIONS_UPPER_BOUND = 1;
+
+ private static final String DEVICE_TEMP_DIR_PATH = "/sdcard/";
+ private static final String FMT_OUTPUT_PREFIX = "output_%1$d_" + System.currentTimeMillis();
+ private static final String FMT_DEVICE_FILENAME = FMT_OUTPUT_PREFIX + "%2$s";
+ private static final String FMT_DEVICE_PATH = DEVICE_TEMP_DIR_PATH + FMT_DEVICE_FILENAME;
+
+ private static final String AM_CMD =
+ "am start -n org.drrickorang.loopback/.LoopbackActivity"
+ + " --ei SF %s --es FileName %s --ei MicSource %s --ei AudioThread %s"
+ + " --ei AudioLevel %s --ei TestType %s --ei BufferTestDuration %s";
+
+ private static final String ERR_PARAMETER_OUT_OF_BOUNDS =
+ "Test parameter '%1$s' is out of bounds. Lower limit = %2$d, upper limit = %3$d";
+
+ private static final String KEY_RESULT_LATENCY_MS = "latency_ms";
+ private static final String KEY_RESULT_LATENCY_CONFIDENCE = "latency_confidence";
+ private static final String KEY_RESULT_RECORDER_BENCHMARK = "recorder_benchmark";
+ private static final String KEY_RESULT_RECORDER_OUTLIER = "recorder_outliers";
+ private static final String KEY_RESULT_PLAYER_BENCHMARK = "player_benchmark";
+ private static final String KEY_RESULT_PLAYER_OUTLIER = "player_outliers";
+ private static final String KEY_RESULT_NUMBER_OF_GLITCHES = "number_of_glitches";
+ private static final String KEY_RESULT_RECORDER_BUFFER_CALLBACK = "late_recorder_callbacks";
+ private static final String KEY_RESULT_PLAYER_BUFFER_CALLBACK = "late_player_callbacks";
+ private static final String KEY_RESULT_GLITCHES_PER_HOUR = "glitches_per_hour";
+ private static final String KEY_RESULT_TEST_STATUS = "test_status";
+ private static final String KEY_RESULT_AUDIO_LEVEL = "audio_level";
+ private static final String KEY_RESULT_RMS = "rms";
+ private static final String KEY_RESULT_RMS_AVERAGE = "rms_average";
+ private static final String KEY_RESULT_SAMPLING_FREQUENCY_CONFIDENCE = "sampling_frequency";
+ private static final String KEY_RESULT_PERIOD_CONFIDENCE = "period_confidence";
+ private static final String KEY_RESULT_SAMPLING_BLOCK_SIZE = "block_size";
+
+ private static final LogFileType[] LATENCY_TEST_LOGS = {
+ LogFileType.RESULT,
+ LogFileType.GRAPH,
+ LogFileType.WAVE,
+ LogFileType.PLAYER_BUFFER,
+ LogFileType.PLAYER_BUFFER_HISTOGRAM,
+ LogFileType.PLAYER_BUFFER_PERIOD_TIMES,
+ LogFileType.RECORDER_BUFFER,
+ LogFileType.RECORDER_BUFFER_HISTOGRAM,
+ LogFileType.RECORDER_BUFFER_PERIOD_TIMES,
+ LogFileType.LOGCAT
+ };
+
+ private static final LogFileType[] GLITCH_TEST_LOGS = {
+ LogFileType.RESULT,
+ LogFileType.GRAPH,
+ LogFileType.WAVE,
+ LogFileType.PLAYER_BUFFER,
+ LogFileType.PLAYER_BUFFER_HISTOGRAM,
+ LogFileType.PLAYER_BUFFER_PERIOD_TIMES,
+ LogFileType.RECORDER_BUFFER,
+ LogFileType.RECORDER_BUFFER_HISTOGRAM,
+ LogFileType.RECORDER_BUFFER_PERIOD_TIMES,
+ LogFileType.GLITCHES_MILLIS,
+ LogFileType.HEAT_MAP,
+ LogFileType.LOGCAT
+ };
+
+ /**
+ * The Audio Latency and Audio Glitch test deals with many various types of log files. To be
+ * able to generate log files in a generic manner, this map is provided to get access to log
+ * file properties like log name prefix, log name file extension and log type (leveraging
+ * tradefed class LogDataType, used when uploading log).
+ */
+ private final synchronized Map<LogFileType, LogFileData> getLogFileDataKeyMap() {
+ if (mFileDataKeyMap != null) {
+ return mFileDataKeyMap;
+ }
+
+ final Map<LogFileType, LogFileData> result = new HashMap<LogFileType, LogFileData>();
+
+ // Populate dictionary with info about various types of logfiles
+ LogFileData l = new LogFileData(".txt", "result", LogDataType.TEXT);
+ result.put(LogFileType.RESULT, l);
+
+ l = new LogFileData(".png", "graph", LogDataType.PNG);
+ result.put(LogFileType.GRAPH, l);
+
+ l = new LogFileData(".wav", "wave", LogDataType.UNKNOWN);
+ result.put(LogFileType.WAVE, l);
+
+ l = new LogFileData("_playerBufferPeriod.txt", "player_buffer", LogDataType.TEXT);
+ result.put(LogFileType.PLAYER_BUFFER, l);
+
+ l = new LogFileData("_playerBufferPeriod.png", "player_buffer_histogram", LogDataType.PNG);
+ result.put(LogFileType.PLAYER_BUFFER_HISTOGRAM, l);
+
+ String fileExtension = "_playerBufferPeriodTimes.txt";
+ String uploadName = "player_buffer_period_times";
+ l = new LogFileData(fileExtension, uploadName, LogDataType.TEXT);
+ result.put(LogFileType.PLAYER_BUFFER_PERIOD_TIMES, l);
+
+ l = new LogFileData("_recorderBufferPeriod.txt", "recorder_buffer", LogDataType.TEXT);
+ result.put(LogFileType.RECORDER_BUFFER, l);
+
+ fileExtension = "_recorderBufferPeriod.png";
+ uploadName = "recorder_buffer_histogram";
+ l = new LogFileData(fileExtension, uploadName, LogDataType.PNG);
+ result.put(LogFileType.RECORDER_BUFFER_HISTOGRAM, l);
+
+ fileExtension = "_recorderBufferPeriodTimes.txt";
+ uploadName = "recorder_buffer_period_times";
+ l = new LogFileData(fileExtension, uploadName, LogDataType.TEXT);
+ result.put(LogFileType.RECORDER_BUFFER_PERIOD_TIMES, l);
+
+ l = new LogFileData("_glitchMillis.txt", "glitches_millis", LogDataType.TEXT);
+ result.put(LogFileType.GLITCHES_MILLIS, l);
+
+
+ l = new LogFileData("_heatMap.png", "heat_map", LogDataType.PNG);
+ result.put(LogFileType.HEAT_MAP, l);
+
+ l = new LogFileData(".txt", "logcat", LogDataType.TEXT);
+ result.put(LogFileType.LOGCAT, l);
+
+ mFileDataKeyMap = Collections.unmodifiableMap(result);
+ return mFileDataKeyMap;
+ }
+
+ private static final Map<String, String> createMetricsKeyMap() {
+ final Map<String, String> result = new HashMap<String, String>();
+
+ result.put("LatencyMs", KEY_RESULT_LATENCY_MS);
+ result.put("LatencyConfidence", KEY_RESULT_LATENCY_CONFIDENCE);
+ result.put("SF", KEY_RESULT_SAMPLING_FREQUENCY_CONFIDENCE);
+ result.put("Recorder Benchmark", KEY_RESULT_RECORDER_BENCHMARK);
+ result.put("Recorder Number of Outliers", KEY_RESULT_RECORDER_OUTLIER);
+ result.put("Player Benchmark", KEY_RESULT_PLAYER_BENCHMARK);
+ result.put("Player Number of Outliers", KEY_RESULT_PLAYER_OUTLIER);
+ result.put("Total Number of Glitches", KEY_RESULT_NUMBER_OF_GLITCHES);
+ result.put("kth% Late Recorder Buffer Callbacks", KEY_RESULT_RECORDER_BUFFER_CALLBACK);
+ result.put("kth% Late Player Buffer Callbacks", KEY_RESULT_PLAYER_BUFFER_CALLBACK);
+ result.put("Glitches Per Hour", KEY_RESULT_GLITCHES_PER_HOUR);
+ result.put("Test Status", KEY_RESULT_TEST_STATUS);
+ result.put("AudioLevel", KEY_RESULT_AUDIO_LEVEL);
+ result.put("RMS", KEY_RESULT_RMS);
+ result.put("Average", KEY_RESULT_RMS_AVERAGE);
+ result.put("PeriodConfidence", KEY_RESULT_PERIOD_CONFIDENCE);
+ result.put("BS", KEY_RESULT_SAMPLING_BLOCK_SIZE);
+
+ return Collections.unmodifiableMap(result);
+ }
+
+ //===================================================================
+ // ENUMS
+ //===================================================================
+ public enum TestType {
+ GLITCH,
+ LATENCY,
+ LATENCY_STRESS,
+ NONE
+ }
+
+ //===================================================================
+ // INNER CLASSES
+ //===================================================================
+ public final class LogFileData {
+ private String fileExtension;
+ private String filePrefix;
+ private LogDataType logDataType;
+
+ private LogFileData(String fileExtension, String filePrefix, LogDataType logDataType) {
+ this.fileExtension = fileExtension;
+ this.filePrefix = filePrefix;
+ this.logDataType = logDataType;
+ }
+ }
+
+ //===================================================================
+ // FUNCTIONS
+ //===================================================================
+
+ /** {@inheritDoc} */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public ITestDevice getDevice() {
+ return mDevice;
+ }
+
+ /**
+ * Test Entry Point
+ *
+ * <p>{@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+
+ initializeTest(listener);
+
+ mTestRunHelper.startTest(1);
+
+ try {
+ if (!verifyTestParameters()) {
+ return;
+ }
+
+ // Stop logcat logging so we can record one logcat log per iteration
+ getDevice().stopLogcat();
+
+ // Run test iterations
+ for (int i = 0; i < mIterations; i++) {
+ CLog.i("---- Iteration " + i + " of " + (mIterations - 1) + " -----");
+
+ final ResultData d = new ResultData();
+ d.setIteration(i);
+ Map<String, String> resultsDictionary = null;
+ resultsDictionary = runTest(d, getSingleTestTimeoutValue());
+
+ mLoopbackTestHelper.addTestData(d, resultsDictionary);
+ }
+
+ mLoopbackTestHelper.processTestData();
+ } finally {
+ Map<String, String> metrics = uploadLogsReturnMetrics(listener);
+ CLog.i("Uploading metrics values:\n" + Arrays.toString(metrics.entrySet().toArray()));
+ mTestRunHelper.endTest(metrics);
+ deleteAllTempFiles();
+ getDevice().startLogcat();
+ }
+ }
+
+ private void initializeTest(ITestInvocationListener listener)
+ throws UnsupportedOperationException, DeviceNotAvailableException {
+
+ mFileDataKeyMap = getLogFileDataKeyMap();
+ TestIdentifier testId = new TestIdentifier(getClass().getCanonicalName(), mRunKey);
+
+ // Allocate helpers
+ mTestRunHelper = new TestRunHelper(listener, testId);
+ mLoopbackTestHelper = new AudioLoopbackTestHelper(mIterations);
+
+ getDevice().disableKeyguard();
+ getDevice().waitForDeviceAvailable(DEVICE_SYNC_MS);
+
+ getDevice().setDate(new Date());
+ CLog.i("syncing device time to host time");
+ }
+
+ private Map<String, String> runTest(ResultData data, final long timeout)
+ throws DeviceNotAvailableException {
+
+ // start measurement and wait for result file
+ final NullOutputReceiver receiver = new NullOutputReceiver();
+
+ final String loopbackCmd = getTestCommand(data.getIteration());
+ CLog.i("Loopback cmd: " + loopbackCmd);
+
+ // Clear logcat
+ // Seems like getDevice().clearLogcat(); doesn't do anything?
+ // Do it through ADB
+ getDevice().executeAdbCommand("logcat", "-c");
+ final long deviceTestStartTime = getDevice().getDeviceDate();
+
+ getDevice()
+ .executeShellCommand(
+ loopbackCmd, receiver, TIMEOUT_MS, TimeUnit.MILLISECONDS, MAX_ATTEMPTS);
+
+ final long loopbackStartTime = System.currentTimeMillis();
+ File loopbackReport = null;
+
+ data.setDeviceTestStartTime(deviceTestStartTime);
+
+ // Try to retrieve result file from device.
+ final String resultFilename = getDeviceFilename(LogFileType.RESULT, data.getIteration());
+ do {
+ RunUtil.getDefault().sleep(POLLING_INTERVAL_MS);
+ if (getDevice().doesFileExist(resultFilename)) {
+ // Store device log files in tmp directory on Host and add to ResultData object
+ storeDeviceFilesOnHost(data);
+ final String reportFilename = data.getLogFile(LogFileType.RESULT);
+ if (reportFilename != null && !reportFilename.isEmpty()) {
+ loopbackReport = new File(reportFilename);
+ if (loopbackReport.length() > 0) {
+ break;
+ }
+ }
+ }
+
+ data.setIsTimedOut(System.currentTimeMillis() - loopbackStartTime >= timeout);
+ } while (!data.hasLogFile(LogFileType.RESULT) && !data.isTimedOut());
+
+ // Grab logcat for iteration
+ final InputStreamSource lc = getDevice().getLogcatSince(deviceTestStartTime);
+ saveLogcatForIteration(data, lc, data.getIteration());
+
+ // Check if test timed out. If so, don't fail the test, but return to upper logic.
+ // We accept certain number of individual test timeouts.
+ if (data.isTimedOut()) {
+ // No device result files retrieved, so no need to parse
+ return null;
+ }
+
+ // parse result
+ Map<String, String> loopbackResult = null;
+
+ try {
+ loopbackResult =
+ AudioLoopbackTestHelper.parseKeyValuePairFromFile(
+ loopbackReport, METRICS_KEY_MAP, mKeyPrefix, "=", "%s: %s");
+ populateResultData(loopbackResult, data);
+
+ // Trust but verify, so get Audio Level from ADB and compare to value from app
+ final int adbAudioLevel =
+ AudioLevelUtility.extractDeviceAudioLevelFromAdbShell(getDevice());
+ if (data.getAudioLevel() != adbAudioLevel) {
+ final String errMsg =
+ String.format(
+ "App Audio Level (%1$d)differs from ADB level (%2$d)",
+ data.getAudioLevel(), adbAudioLevel);
+ mTestRunHelper.reportFailure(errMsg);
+ }
+ } catch (final IOException ioe) {
+ CLog.e(ioe);
+ mTestRunHelper.reportFailure("I/O error while parsing Loopback result.");
+ } catch (final NumberFormatException ne) {
+ CLog.e(ne);
+ mTestRunHelper.reportFailure("Number format error parsing Loopback result.");
+ }
+
+ return loopbackResult;
+ }
+
+ private String getMetricsKey(final String key) {
+ return mKeyPrefix + key;
+ }
+ private final long getSingleTestTimeoutValue() {
+ return Long.parseLong(mBufferTestDuration) * 1000 + TIMEOUT_MS;
+ }
+
+ private Map<String, String> uploadLogsReturnMetrics(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+
+ // "resultDictionary" is used to post results to dashboards like BlackBox
+ // "results" contains test logs to be uploaded; i.e. to Sponge
+
+ List<ResultData> results = null;
+ Map<String, String> resultDictionary = new HashMap<String, String>();
+
+ switch (getTestType()) {
+ case GLITCH:
+ resultDictionary = mLoopbackTestHelper.getResultDictionaryForIteration(0);
+ // Upload all test files to be backward compatible with old test
+ results = mLoopbackTestHelper.getAllTestData();
+ break;
+ case LATENCY:
+ {
+ final int nrOfValidResults = mLoopbackTestHelper.processTestData();
+ if (nrOfValidResults == 0) {
+ mTestRunHelper.reportFailure("No good data was collected");
+ } else {
+ // use dictionary collected from single test run
+ resultDictionary = mLoopbackTestHelper.getResultDictionaryForIteration(0);
+ }
+
+ // Upload all test files to be backward compatible with old test
+ results = mLoopbackTestHelper.getAllTestData();
+ }
+ break;
+ case LATENCY_STRESS:
+ {
+ final int nrOfValidResults = mLoopbackTestHelper.processTestData();
+ if (nrOfValidResults == 0) {
+ mTestRunHelper.reportFailure("No good data was collected");
+ } else {
+ mLoopbackTestHelper.populateStressTestMetrics(resultDictionary, mKeyPrefix);
+ }
+
+ results = mLoopbackTestHelper.getWorstResults(MAX_NR_OF_LOG_UPLOADS);
+
+ // Save all test data in a spreadsheet style csv file for post test analysis
+ try {
+ saveResultsAsCSVFile(listener);
+ } catch (final IOException e) {
+ CLog.e(e);
+ }
+ }
+ break;
+ default:
+ break;
+ }
+
+ // Upload relevant logs
+ for (final ResultData d : results) {
+ final LogFileType[] logFileTypes = getLogFileTypesForCurrentTest();
+ for (final LogFileType logType : logFileTypes) {
+ uploadLog(listener, logType, d);
+ }
+ }
+
+ return resultDictionary;
+ }
+
+ private TestType getTestType() {
+ if (mTestType.equals(TESTTYPE_GLITCH_STR)) {
+ if (GLITCH_ITERATIONS_LOWER_BOUND <= mIterations
+ && mIterations <= GLITCH_ITERATIONS_UPPER_BOUND) {
+ return TestType.GLITCH;
+ }
+ }
+
+ if (mTestType.equals(TESTTYPE_LATENCY_STR)) {
+ if (mIterations == 1) {
+ return TestType.LATENCY;
+ }
+
+ if (LATENCY_ITERATIONS_LOWER_BOUND <= mIterations
+ && mIterations <= LATENCY_ITERATIONS_UPPER_BOUND) {
+ return TestType.LATENCY_STRESS;
+ }
+ }
+
+ return TestType.NONE;
+ }
+
+ private boolean verifyTestParameters() {
+ if (getTestType() != TestType.NONE) {
+ return true;
+ }
+
+ if (mTestType.equals(TESTTYPE_GLITCH_STR)
+ && (mIterations < GLITCH_ITERATIONS_LOWER_BOUND
+ || mIterations > GLITCH_ITERATIONS_UPPER_BOUND)) {
+ final String error =
+ String.format(
+ ERR_PARAMETER_OUT_OF_BOUNDS,
+ "iterations",
+ GLITCH_ITERATIONS_LOWER_BOUND,
+ GLITCH_ITERATIONS_UPPER_BOUND);
+ mTestRunHelper.reportFailure(error);
+ return false;
+ }
+
+ if (mTestType.equals(TESTTYPE_LATENCY_STR)
+ && (mIterations < LATENCY_ITERATIONS_LOWER_BOUND
+ || mIterations > LATENCY_ITERATIONS_UPPER_BOUND)) {
+ final String error =
+ String.format(
+ ERR_PARAMETER_OUT_OF_BOUNDS,
+ "iterations",
+ LATENCY_ITERATIONS_LOWER_BOUND,
+ LATENCY_ITERATIONS_UPPER_BOUND);
+ mTestRunHelper.reportFailure(error);
+ return false;
+ }
+
+ return true;
+ }
+
+ private void populateResultData(final Map<String, String> results, ResultData data) {
+ if (results == null || results.isEmpty()) {
+ return;
+ }
+
+ String key = getMetricsKey(KEY_RESULT_LATENCY_MS);
+ if (results.containsKey(key)) {
+ data.setLatency(Float.parseFloat(results.get(key)));
+ }
+
+ key = getMetricsKey(KEY_RESULT_LATENCY_CONFIDENCE);
+ if (results.containsKey(key)) {
+ data.setConfidence(Float.parseFloat(results.get(key)));
+ }
+
+ key = getMetricsKey(KEY_RESULT_AUDIO_LEVEL);
+ if (results.containsKey(key)) {
+ data.setAudioLevel(Integer.parseInt(results.get(key)));
+ }
+
+ key = getMetricsKey(KEY_RESULT_RMS);
+ if (results.containsKey(key)) {
+ data.setRMS(Float.parseFloat(results.get(key)));
+ }
+
+ key = getMetricsKey(KEY_RESULT_RMS_AVERAGE);
+ if (results.containsKey(key)) {
+ data.setRMSAverage(Float.parseFloat(results.get(key)));
+ }
+
+ key = getMetricsKey(KEY_RESULT_PERIOD_CONFIDENCE);
+ if (results.containsKey(key)) {
+ data.setPeriodConfidence(Float.parseFloat(results.get(key)));
+ }
+
+ key = getMetricsKey(KEY_RESULT_SAMPLING_BLOCK_SIZE);
+ if (results.containsKey(key)) {
+ data.setBlockSize(Integer.parseInt(results.get(key)));
+ }
+ }
+
+ private void storeDeviceFilesOnHost(ResultData data) throws DeviceNotAvailableException {
+ final int iteration = data.getIteration();
+ for (final LogFileType log : getLogFileTypesForCurrentTest()) {
+ if (getDevice().doesFileExist(getDeviceFilename(log, iteration))) {
+ final String deviceFileName = getDeviceFilename(log, iteration);
+ final File logFile = getDevice().pullFile(deviceFileName);
+ data.setLogFile(log, logFile.getAbsolutePath());
+ CLog.i("Delete file from device: " + deviceFileName);
+ deleteFileFromDevice(deviceFileName);
+ }
+ }
+ }
+
+ private void deleteAllTempFiles() {
+ for (final ResultData d : mLoopbackTestHelper.getAllTestData()) {
+ final LogFileType[] logFileTypes = getLogFileTypesForCurrentTest();
+ for (final LogFileType logType : logFileTypes) {
+ final String logFilename = d.getLogFile(logType);
+ if (logFilename == null || logFilename.isEmpty()) {
+ CLog.e("Logfile not found for LogFileType=" + logType.name());
+ } else {
+ FileUtil.deleteFile(new File(logFilename));
+ }
+ }
+ }
+ }
+
+ private void deleteFileFromDevice(String deviceFileName) throws DeviceNotAvailableException {
+ getDevice().executeShellCommand("rm -f " + deviceFileName);
+ }
+
+ private final LogFileType[] getLogFileTypesForCurrentTest() {
+ switch (getTestType()) {
+ case GLITCH:
+ return GLITCH_TEST_LOGS;
+ case LATENCY:
+ case LATENCY_STRESS:
+ return LATENCY_TEST_LOGS;
+ default:
+ return null;
+ }
+ }
+
+ private String getKeyPrefixForIteration(int iteration) {
+ if (mIterations == 1) {
+ // If only one run, skip the iteration number
+ return mKeyPrefix;
+ }
+ return mKeyPrefix + iteration + "_";
+ }
+
+ private String getDeviceFilename(LogFileType key, int iteration) {
+ final Map<LogFileType, LogFileData> map = getLogFileDataKeyMap();
+ if (map.containsKey(key)) {
+ final LogFileData data = map.get(key);
+ return String.format(FMT_DEVICE_PATH, iteration, data.fileExtension);
+ }
+ return null;
+ }
+
+ private void uploadLog(ITestInvocationListener listener, LogFileType key, ResultData data) {
+ final Map<LogFileType, LogFileData> map = getLogFileDataKeyMap();
+ if (!map.containsKey(key)) {
+ return;
+ }
+
+ final LogFileData logInfo = map.get(key);
+ final String prefix = getKeyPrefixForIteration(data.getIteration()) + logInfo.filePrefix;
+ final LogDataType logDataType = logInfo.logDataType;
+ final String logFilename = data.getLogFile(key);
+ if (logFilename == null || logFilename.isEmpty()) {
+ CLog.e("Logfile not found for LogFileType=" + key.name());
+ } else {
+ File logFile = new File(logFilename);
+ InputStreamSource iss = new FileInputStreamSource(logFile);
+ listener.testLog(prefix, logDataType, iss);
+
+ // cleanup
+ iss.cancel();
+ }
+ }
+
+ private void saveLogcatForIteration(ResultData data, InputStreamSource logcat, int iteration) {
+ if (logcat == null) {
+ CLog.i("Logcat could not be saved for iteration " + iteration);
+ return;
+ }
+
+ //create a temp file
+ File temp;
+ try {
+ temp = FileUtil.createTempFile("logcat_" + iteration + "_", ".txt");
+ data.setLogFile(LogFileType.LOGCAT, temp.getAbsolutePath());
+
+ // Copy logcat data into temp file
+ Files.copy(logcat.createInputStream(), temp.toPath(), REPLACE_EXISTING);
+ logcat.cancel();
+ } catch (final IOException e) {
+ CLog.i("Error when saving logcat for iteration=" + iteration);
+ CLog.e(e);
+ }
+ }
+
+ private void saveResultsAsCSVFile(ITestInvocationListener listener)
+ throws DeviceNotAvailableException, IOException {
+ final File csvTmpFile = File.createTempFile("audio_test_data", "csv");
+ mLoopbackTestHelper.writeAllResultsToCSVFile(csvTmpFile, getDevice());
+ InputStreamSource iss = new FileInputStreamSource(csvTmpFile);
+ listener.testLog("audio_test_data", LogDataType.JACOCO_CSV, iss);
+
+ // cleanup
+ iss.cancel();
+ csvTmpFile.delete();
+ }
+
+ private String getTestCommand(int currentIteration) {
+ return String.format(
+ AM_CMD,
+ mSamplingFreq,
+ String.format(FMT_OUTPUT_PREFIX, currentIteration),
+ mMicSource,
+ mAudioThread,
+ mAudioLevel,
+ mTestType,
+ mBufferTestDuration);
+ }
+}
diff --git a/src/com/android/media/tests/AudioLoopbackTestHelper.java b/src/com/android/media/tests/AudioLoopbackTestHelper.java
new file mode 100644
index 0000000..4e9c2b0
--- /dev/null
+++ b/src/com/android/media/tests/AudioLoopbackTestHelper.java
@@ -0,0 +1,608 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.media.tests;
+
+import com.android.media.tests.AudioLoopbackImageAnalyzer.Result;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.util.Pair;
+
+import com.google.common.io.Files;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Helper class for AudioLoopbackTest. It keeps runtime data, analytics, */
+public class AudioLoopbackTestHelper {
+
+ private StatisticsData mLatencyStats = null;
+ private StatisticsData mConfidenceStats = null;
+ private ArrayList<ResultData> mAllResults;
+ private ArrayList<ResultData> mGoodResults = new ArrayList<ResultData>();
+ private ArrayList<ResultData> mBadResults = new ArrayList<ResultData>();
+ private ArrayList<Map<String, String>> mResultDictionaries =
+ new ArrayList<Map<String, String>>();
+
+ // Controls acceptable tolerance in ms around median latency
+ private static final double TOLERANCE = 2.0;
+
+ //===================================================================
+ // ENUMS
+ //===================================================================
+ public enum LogFileType {
+ RESULT,
+ WAVE,
+ GRAPH,
+ PLAYER_BUFFER,
+ PLAYER_BUFFER_HISTOGRAM,
+ PLAYER_BUFFER_PERIOD_TIMES,
+ RECORDER_BUFFER,
+ RECORDER_BUFFER_HISTOGRAM,
+ RECORDER_BUFFER_PERIOD_TIMES,
+ GLITCHES_MILLIS,
+ HEAT_MAP,
+ LOGCAT
+ }
+
+ //===================================================================
+ // INNER CLASSES
+ //===================================================================
+ private class StatisticsData {
+ double mMin = 0;
+ double mMax = 0;
+ double mMean = 0;
+ double mMedian = 0;
+
+ @Override
+ public String toString() {
+ return String.format(
+ "min = %1$f, max = %2$f, median=%3$f, mean = %4$f", mMin, mMax, mMedian, mMean);
+ }
+ }
+
+ /** ResultData is an inner class that holds results and logfile info from each test run */
+ public static class ResultData {
+ private Float mLatencyMs;
+ private Float mLatencyConfidence;
+ private Integer mAudioLevel;
+ private Integer mIteration;
+ private Long mDeviceTestStartTime;
+ private boolean mIsTimedOut = false;
+ private HashMap<LogFileType, String> mLogs = new HashMap<LogFileType, String>();
+ private Result mImageAnalyzerResult = Result.UNKNOWN;
+ private String mFailureReason = null;
+
+ // Optional
+ private Float mPeriodConfidence = Float.valueOf(0.0f);
+ private Float mRms = Float.valueOf(0.0f);
+ private Float mRmsAverage = Float.valueOf(0.0f);
+ private Integer mBblockSize = Integer.valueOf(0);
+
+ public float getLatency() {
+ return mLatencyMs.floatValue();
+ }
+
+ public void setLatency(float latencyMs) {
+ this.mLatencyMs = Float.valueOf(latencyMs);
+ }
+
+ public float getConfidence() {
+ return mLatencyConfidence.floatValue();
+ }
+
+ public void setConfidence(float latencyConfidence) {
+ this.mLatencyConfidence = Float.valueOf(latencyConfidence);
+ }
+
+ public float getPeriodConfidence() {
+ return mPeriodConfidence.floatValue();
+ }
+
+ public void setPeriodConfidence(float periodConfidence) {
+ this.mPeriodConfidence = Float.valueOf(periodConfidence);
+ }
+
+ public float getRMS() {
+ return mRms.floatValue();
+ }
+
+ public void setRMS(float rms) {
+ this.mRms = Float.valueOf(rms);
+ }
+
+ public float getRMSAverage() {
+ return mRmsAverage.floatValue();
+ }
+
+ public void setRMSAverage(float rmsAverage) {
+ this.mRmsAverage = Float.valueOf(rmsAverage);
+ }
+
+ public int getAudioLevel() {
+ return mAudioLevel.intValue();
+ }
+
+ public void setAudioLevel(int audioLevel) {
+ this.mAudioLevel = Integer.valueOf(audioLevel);
+ }
+
+ public int getBlockSize() {
+ return mBblockSize.intValue();
+ }
+
+ public void setBlockSize(int blockSize) {
+ this.mBblockSize = Integer.valueOf(blockSize);
+ }
+
+ public int getIteration() {
+ return mIteration.intValue();
+ }
+
+ public void setIteration(int iteration) {
+ this.mIteration = Integer.valueOf(iteration);
+ }
+
+ public long getDeviceTestStartTime() {
+ return mDeviceTestStartTime.longValue();
+ }
+
+ public void setDeviceTestStartTime(long deviceTestStartTime) {
+ this.mDeviceTestStartTime = Long.valueOf(deviceTestStartTime);
+ }
+
+ public Result getImageAnalyzerResult() {
+ return mImageAnalyzerResult;
+ }
+
+ public void setImageAnalyzerResult(Result imageAnalyzerResult) {
+ this.mImageAnalyzerResult = imageAnalyzerResult;
+ }
+
+ public String getFailureReason() {
+ return mFailureReason;
+ }
+
+ public void setFailureReason(String failureReason) {
+ this.mFailureReason = failureReason;
+ }
+
+ public boolean isTimedOut() {
+ return mIsTimedOut;
+ }
+
+ public void setIsTimedOut(boolean isTimedOut) {
+ this.mIsTimedOut = isTimedOut;
+ }
+
+ public String getLogFile(LogFileType log) {
+ return mLogs.get(log);
+ }
+
+ public void setLogFile(LogFileType log, String filename) {
+ CLog.i("setLogFile: type=" + log.name() + ", filename=" + filename);
+ if (!mLogs.containsKey(log) && filename != null && !filename.isEmpty()) {
+ mLogs.put(log, filename);
+ }
+ }
+
+ public boolean hasBadResults() {
+ return hasTimedOut()
+ || hasNoTestResults()
+ || hasNoLatencyResult()
+ || hasNoLatencyConfidence()
+ || mImageAnalyzerResult == Result.FAIL;
+ }
+
+ public boolean hasTimedOut() {
+ return mIsTimedOut;
+ }
+
+ public boolean hasLogFile(LogFileType log) {
+ return mLogs.containsKey(log);
+ }
+
+ public boolean hasNoLatencyResult() {
+ return mLatencyMs == null;
+ }
+
+ public boolean hasNoLatencyConfidence() {
+ return mLatencyConfidence == null;
+ }
+
+ public boolean hasNoTestResults() {
+ return hasNoLatencyConfidence() && hasNoLatencyResult();
+ }
+
+ public static Comparator<ResultData> latencyComparator =
+ new Comparator<ResultData>() {
+ @Override
+ public int compare(ResultData o1, ResultData o2) {
+ return o1.mLatencyMs.compareTo(o2.mLatencyMs);
+ }
+ };
+
+ public static Comparator<ResultData> confidenceComparator =
+ new Comparator<ResultData>() {
+ @Override
+ public int compare(ResultData o1, ResultData o2) {
+ return o1.mLatencyConfidence.compareTo(o2.mLatencyConfidence);
+ }
+ };
+
+ public static Comparator<ResultData> iteratorComparator =
+ new Comparator<ResultData>() {
+ @Override
+ public int compare(ResultData o1, ResultData o2) {
+ return Integer.compare(o1.mIteration, o2.mIteration);
+ }
+ };
+
+ @Override
+ public String toString() {
+ final String NL = "\n";
+ final StringBuilder sb = new StringBuilder(512);
+ sb.append("{").append(NL);
+ sb.append("{\nlatencyMs=").append(mLatencyMs).append(NL);
+ sb.append("latencyConfidence=").append(mLatencyConfidence).append(NL);
+ sb.append("isTimedOut=").append(mIsTimedOut).append(NL);
+ sb.append("iteration=").append(mIteration).append(NL);
+ sb.append("logs=").append(Arrays.toString(mLogs.values().toArray())).append(NL);
+ sb.append("audioLevel=").append(mAudioLevel).append(NL);
+ sb.append("deviceTestStartTime=").append(mDeviceTestStartTime).append(NL);
+ sb.append("rms=").append(mRms).append(NL);
+ sb.append("rmsAverage=").append(mRmsAverage).append(NL);
+ sb.append("}").append(NL);
+ return sb.toString();
+ }
+ }
+
+ public AudioLoopbackTestHelper(int iterations) {
+ mAllResults = new ArrayList<ResultData>(iterations);
+ }
+
+ public void addTestData(ResultData data, Map<String, String> resultDictionary) {
+ mResultDictionaries.add(data.getIteration(), resultDictionary);
+ mAllResults.add(data);
+
+ // Analyze captured screenshot to see if wave form is within reason
+ final String screenshot = data.getLogFile(LogFileType.GRAPH);
+ final Pair<Result, String> result = AudioLoopbackImageAnalyzer.analyzeImage(screenshot);
+ data.setImageAnalyzerResult(result.first);
+ data.setFailureReason(result.second);
+ }
+
+ public final List<ResultData> getAllTestData() {
+ return mAllResults;
+ }
+
+ public Map<String, String> getResultDictionaryForIteration(int i) {
+ return mResultDictionaries.get(i);
+ }
+
+ /**
+ * Returns a list of the worst test result objects, up to maxNrOfWorstResults
+ *
+ * <p>
+ *
+ * <ol>
+ * <li> Tests in the bad results list are added first
+ * <li> If still space, add test results based on low confidence and then tests that are
+ * outside tolerance boundaries
+ * </ol>
+ *
+ * @param maxNrOfWorstResults
+ * @return list of worst test result objects
+ */
+ public List<ResultData> getWorstResults(int maxNrOfWorstResults) {
+ int counter = 0;
+ final ArrayList<ResultData> worstResults = new ArrayList<ResultData>(maxNrOfWorstResults);
+
+ for (final ResultData data : mBadResults) {
+ if (counter < maxNrOfWorstResults) {
+ worstResults.add(data);
+ counter++;
+ }
+ }
+
+ for (final ResultData data : mGoodResults) {
+ if (counter < maxNrOfWorstResults) {
+ boolean failed = false;
+ if (data.getConfidence() < 1.0f) {
+ data.setFailureReason("Low confidence");
+ failed = true;
+ } else if (data.getLatency() < (mLatencyStats.mMedian - TOLERANCE)
+ || data.getLatency() > (mLatencyStats.mMedian + TOLERANCE)) {
+ data.setFailureReason("Latency not within tolerance from median");
+ failed = true;
+ }
+
+ if (failed) {
+ worstResults.add(data);
+ counter++;
+ }
+ }
+ }
+
+ return worstResults;
+ }
+
+ public static Map<String, String> parseKeyValuePairFromFile(
+ File result,
+ final Map<String, String> dictionary,
+ final String resultKeyPrefix,
+ final String splitOn,
+ final String keyValueFormat)
+ throws IOException {
+
+ final Map<String, String> resultMap = new HashMap<String, String>();
+ final BufferedReader br = Files.newReader(result, StandardCharsets.UTF_8);
+
+ try {
+ String line = br.readLine();
+ while (line != null) {
+ line = line.trim().replaceAll(" +", " ");
+ final String[] tokens = line.split(splitOn);
+ if (tokens.length >= 2) {
+ final String key = tokens[0].trim();
+ final String value = tokens[1].trim();
+ if (dictionary.containsKey(key)) {
+ CLog.i(String.format(keyValueFormat, key, value));
+ resultMap.put(resultKeyPrefix + dictionary.get(key), value);
+ }
+ }
+ line = br.readLine();
+ }
+ } finally {
+ br.close();
+ }
+ return resultMap;
+ }
+
+ public int processTestData() {
+
+ // Collect statistics about the test run
+ int nrOfValidResults = 0;
+ double sumLatency = 0;
+ double sumConfidence = 0;
+
+ final int totalNrOfTests = mAllResults.size();
+ mLatencyStats = new StatisticsData();
+ mConfidenceStats = new StatisticsData();
+ mBadResults = new ArrayList<ResultData>();
+ mGoodResults = new ArrayList<ResultData>(totalNrOfTests);
+
+ // Copy all results into Good results list
+ mGoodResults.addAll(mAllResults);
+
+ for (final ResultData data : mAllResults) {
+ if (data.hasBadResults()) {
+ mBadResults.add(data);
+ continue;
+ }
+ // Get mean values
+ sumLatency += data.getLatency();
+ sumConfidence += data.getConfidence();
+ }
+
+ if (!mBadResults.isEmpty()) {
+ analyzeBadResults(mBadResults, mAllResults.size());
+ }
+
+ // Remove bad runs from result array
+ mGoodResults.removeAll(mBadResults);
+
+ // Fail test immediately if we don't have ANY good results
+ if (mGoodResults.isEmpty()) {
+ return 0;
+ }
+
+ nrOfValidResults = mGoodResults.size();
+
+ // ---- LATENCY: Get Median, Min and Max values ----
+ Collections.sort(mGoodResults, ResultData.latencyComparator);
+
+ mLatencyStats.mMin = mGoodResults.get(0).mLatencyMs;
+ mLatencyStats.mMax = mGoodResults.get(nrOfValidResults - 1).mLatencyMs;
+ mLatencyStats.mMean = sumLatency / nrOfValidResults;
+ // Is array even or odd numbered
+ if (nrOfValidResults % 2 == 0) {
+ final int middle = nrOfValidResults / 2;
+ final float middleLeft = mGoodResults.get(middle - 1).mLatencyMs;
+ final float middleRight = mGoodResults.get(middle).mLatencyMs;
+ mLatencyStats.mMedian = (middleLeft + middleRight) / 2.0f;
+ } else {
+ // It's and odd numbered array, just grab the middle value
+ mLatencyStats.mMedian = mGoodResults.get(nrOfValidResults / 2).mLatencyMs;
+ }
+
+ // ---- CONFIDENCE: Get Median, Min and Max values ----
+ Collections.sort(mGoodResults, ResultData.confidenceComparator);
+
+ mConfidenceStats.mMin = mGoodResults.get(0).mLatencyConfidence;
+ mConfidenceStats.mMax = mGoodResults.get(nrOfValidResults - 1).mLatencyConfidence;
+ mConfidenceStats.mMean = sumConfidence / nrOfValidResults;
+ // Is array even or odd numbered
+ if (nrOfValidResults % 2 == 0) {
+ final int middle = nrOfValidResults / 2;
+ final float middleLeft = mGoodResults.get(middle - 1).mLatencyConfidence;
+ final float middleRight = mGoodResults.get(middle).mLatencyConfidence;
+ mConfidenceStats.mMedian = (middleLeft + middleRight) / 2.0f;
+ } else {
+ // It's and odd numbered array, just grab the middle value
+ mConfidenceStats.mMedian = mGoodResults.get(nrOfValidResults / 2).mLatencyConfidence;
+ }
+
+ for (final ResultData data : mGoodResults) {
+ // Check if within Latency Tolerance
+ if (data.getConfidence() < 1.0f) {
+ data.setFailureReason("Low confidence");
+ } else if (data.getLatency() < (mLatencyStats.mMedian - TOLERANCE)
+ || data.getLatency() > (mLatencyStats.mMedian + TOLERANCE)) {
+ data.setFailureReason("Latency not within tolerance from median");
+ }
+ }
+
+ // Create histogram
+ // Strategy: Create buckets based on whole ints, like 16 ms, 17 ms, 18 ms etc. Count how
+ // many tests fall into each bucket. Just cast the float to an int, no rounding up/down
+ // required.
+ final int[] histogram = new int[(int) mLatencyStats.mMax + 1];
+ for (final ResultData rd : mGoodResults) {
+ // Increase value in bucket
+ histogram[(int) (rd.mLatencyMs.floatValue())]++;
+ }
+
+ CLog.i("========== VALID RESULTS ============================================");
+ CLog.i(String.format("Valid tests: %1$d of %2$d", nrOfValidResults, totalNrOfTests));
+ CLog.i("Latency: " + mLatencyStats.toString());
+ CLog.i("Confidence: " + mConfidenceStats.toString());
+ CLog.i("========== HISTOGRAM ================================================");
+ for (int i = 0; i < histogram.length; i++) {
+ if (histogram[i] > 0) {
+ CLog.i(String.format("%1$01d ms => %2$d", i, histogram[i]));
+ }
+ }
+
+ // VERIFY the good results by running image analysis on the
+ // screenshot of the incoming audio waveform
+
+ return nrOfValidResults;
+ }
+
+ public void writeAllResultsToCSVFile(File csvFile, ITestDevice device)
+ throws DeviceNotAvailableException, FileNotFoundException,
+ UnsupportedEncodingException {
+
+ final String deviceType = device.getProperty("ro.build.product");
+ final String buildId = device.getBuildAlias();
+ final String serialNumber = device.getSerialNumber();
+
+ // Sort data on iteration
+ Collections.sort(mAllResults, ResultData.iteratorComparator);
+
+ final StringBuilder sb = new StringBuilder(256);
+ final PrintWriter writer = new PrintWriter(csvFile, StandardCharsets.UTF_8.name());
+ final String SEPARATOR = ",";
+
+ // Write column labels
+ writer.println(
+ "Device Time,Device Type,Build Id,Serial Number,Iteration,Latency,"
+ + "Confidence,Period Confidence,Block Size,Audio Level,RMS,RMS Average,"
+ + "Image Analysis,Failure Reason");
+ for (final ResultData data : mAllResults) {
+ final Instant instant = Instant.ofEpochSecond(data.mDeviceTestStartTime);
+
+ sb.append(instant).append(SEPARATOR);
+ sb.append(deviceType).append(SEPARATOR);
+ sb.append(buildId).append(SEPARATOR);
+ sb.append(serialNumber).append(SEPARATOR);
+ sb.append(data.getIteration()).append(SEPARATOR);
+ sb.append(data.getLatency()).append(SEPARATOR);
+ sb.append(data.getConfidence()).append(SEPARATOR);
+ sb.append(data.getPeriodConfidence()).append(SEPARATOR);
+ sb.append(data.getBlockSize()).append(SEPARATOR);
+ sb.append(data.getAudioLevel()).append(SEPARATOR);
+ sb.append(data.getRMS()).append(SEPARATOR);
+ sb.append(data.getRMSAverage()).append(SEPARATOR);
+ sb.append(data.getImageAnalyzerResult().name()).append(SEPARATOR);
+ sb.append(data.getFailureReason());
+
+ writer.println(sb.toString());
+
+ sb.setLength(0);
+ }
+ writer.close();
+ }
+
+ private void analyzeBadResults(ArrayList<ResultData> badResults, int totalNrOfTests) {
+ int testNoData = 0;
+ int testTimeoutCounts = 0;
+ int testResultsNotFoundCounts = 0;
+ int testWithoutLatencyResultCount = 0;
+ int testWithoutConfidenceResultCount = 0;
+
+ for (final ResultData data : badResults) {
+ if (data.hasTimedOut()) {
+ testTimeoutCounts++;
+ testNoData++;
+ continue;
+ }
+
+ if (data.hasNoTestResults()) {
+ testResultsNotFoundCounts++;
+ testNoData++;
+ continue;
+ }
+
+ if (data.hasNoLatencyResult()) {
+ testWithoutLatencyResultCount++;
+ testNoData++;
+ continue;
+ }
+
+ if (data.hasNoLatencyConfidence()) {
+ testWithoutConfidenceResultCount++;
+ testNoData++;
+ continue;
+ }
+ }
+
+ CLog.i("========== BAD RESULTS ============================================");
+ CLog.i(String.format("No Data: %1$d of %2$d", testNoData, totalNrOfTests));
+ CLog.i(String.format("Timed out: %1$d of %2$d", testTimeoutCounts, totalNrOfTests));
+ CLog.i(
+ String.format(
+ "No results: %1$d of %2$d", testResultsNotFoundCounts, totalNrOfTests));
+ CLog.i(
+ String.format(
+ "No Latency results: %1$d of %2$d",
+ testWithoutLatencyResultCount, totalNrOfTests));
+ CLog.i(
+ String.format(
+ "No Confidence results: %1$d of %2$d",
+ testWithoutConfidenceResultCount, totalNrOfTests));
+ }
+
+ /** Generates metrics dictionary for stress test */
+ public void populateStressTestMetrics(
+ Map<String, String> metrics, final String resultKeyPrefix) {
+ metrics.put(resultKeyPrefix + "total_nr_of_tests", Integer.toString(mAllResults.size()));
+ metrics.put(resultKeyPrefix + "nr_of_good_tests", Integer.toString(mGoodResults.size()));
+ metrics.put(resultKeyPrefix + "latency_max", Double.toString(mLatencyStats.mMax));
+ metrics.put(resultKeyPrefix + "latency_min", Double.toString(mLatencyStats.mMin));
+ metrics.put(resultKeyPrefix + "latency_mean", Double.toString(mLatencyStats.mMean));
+ metrics.put(resultKeyPrefix + "latency_median", Double.toString(mLatencyStats.mMedian));
+ metrics.put(resultKeyPrefix + "confidence_max", Double.toString(mConfidenceStats.mMax));
+ metrics.put(resultKeyPrefix + "confidence_min", Double.toString(mConfidenceStats.mMin));
+ metrics.put(resultKeyPrefix + "confidence_mean", Double.toString(mConfidenceStats.mMean));
+ metrics.put(
+ resultKeyPrefix + "confidence_median", Double.toString(mConfidenceStats.mMedian));
+ }
+}
diff --git a/src/com/android/media/tests/Camera2FrameworkStressTest.java b/src/com/android/media/tests/Camera2FrameworkStressTest.java
new file mode 100644
index 0000000..8d62595
--- /dev/null
+++ b/src/com/android/media/tests/Camera2FrameworkStressTest.java
@@ -0,0 +1,188 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.IFileEntry;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.util.FileUtil;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Camera2 framework stress test
+ * This is a test invocation that runs stress tests against Camera2 framework (API & HAL) to
+ * isolate stability issues from Camera application.
+ * This invocation uses a Camera2InstrumentationTestRunner to run a set of
+ * Camera framework stress tests.
+ */
+@OptionClass(alias = "camera2-framework-stress")
+public class Camera2FrameworkStressTest extends CameraTestBase {
+
+ // Keys in instrumentation test metrics
+ private static final String RESULT_DIR = "/sdcard/camera-out/";
+ private static final String RESULT_FILE_FORMAT = RESULT_DIR + "fwk-stress_camera_%s.txt";
+ private static final Pattern RESULT_FILE_REGEX = Pattern.compile(
+ "^fwk-stress_camera_(?<id>.+).txt");
+ private static final String KEY_NUM_ATTEMPTS = "numAttempts";
+ private static final String KEY_ITERATION = "iteration";
+
+ public Camera2FrameworkStressTest() {
+ // Note that default value in constructor will be overridden by the passing option from
+ // a command line.
+ setTestPackage("com.android.mediaframeworktest");
+ setTestRunner("com.android.mediaframeworktest.Camera2InstrumentationTestRunner");
+ setRuKey("CameraFrameworkStress");
+ setTestTimeoutMs(2 * 60 * 60 * 1000); // 2 hours
+ setLogcatOnFailure(true);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ runInstrumentationTest(listener, new CollectingListener(listener));
+ }
+
+ /**
+ * A listener to collect the output from test run and fatal errors
+ */
+ private class CollectingListener extends DefaultCollectingListener {
+
+ public CollectingListener(ITestInvocationListener listener) {
+ super(listener);
+ }
+
+ @Override
+ public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) {
+ if (testMetrics == null) {
+ return; // No-op if there is nothing to post.
+ }
+ for (Map.Entry<String, String> metric : testMetrics.entrySet()) {
+ getAggregatedMetrics().put(metric.getKey(), metric.getValue());
+ }
+ }
+
+ @Override
+ public void testEnded(TestIdentifier test, long endTime, Map<String, String> testMetrics) {
+ if (hasTestRunFatalError()) {
+ CLog.v("The instrumentation result not found. Fall back to get the metrics from a "
+ + "log file. errorMsg: %s", getCollectingListener().getErrorMessage());
+ }
+
+ // For stress test, parse the metrics from a log file.
+ testMetrics = parseLog(test.getTestName());
+ super.testEnded(test, endTime, testMetrics);
+ }
+
+ // Return null if failed to parse the result file or the test didn't even start.
+ private Map<String, String> parseLog(String testName) {
+ Map<String, String> postMetrics = new HashMap<String, String>();
+ String resultFilePath;
+ try {
+ for (String cameraId : getCameraIdList(RESULT_DIR)) {
+ File outputFile = FileUtil.createTempFile("fwk-stress", ".txt");
+ resultFilePath = getResultFilePath(cameraId);
+ getDevice().pullFile(resultFilePath, outputFile);
+ if (outputFile == null) {
+ throw new DeviceNotAvailableException(String.format("Failed to pull the "
+ + "result file: %s", resultFilePath),
+ getDevice().getSerialNumber());
+ }
+
+ BufferedReader reader = new BufferedReader(new FileReader(outputFile));
+ String line;
+ Map<String, String> resultMap = new HashMap<String, String>();
+
+ // Parse results from log file that contain the key-value pairs.
+ // eg. "numAttempts=10|iteration=9[|cameraId=0]"
+ try {
+ while ((line = reader.readLine()) != null) {
+ String[] pairs = line.split("\\|");
+ for (String pair : pairs) {
+ String[] keyValue = pair.split("=");
+ // Each should be a pair of key and value.
+ String key = keyValue[0].trim();
+ String value = keyValue[1].trim();
+ resultMap.put(key, value);
+ }
+ }
+ } finally {
+ reader.close();
+ }
+
+ // Fail if a stress test doesn't start.
+ if (0 == Integer.parseInt(resultMap.get(KEY_NUM_ATTEMPTS))) {
+ CLog.w("Failed to start stress tests. test setup configured incorrectly?");
+ return null;
+ }
+ // Post the number of iterations only with the key name associated with
+ // test name and camera ID.
+ String keyName = testName + "_" + cameraId;
+ postMetrics.put(keyName, resultMap.get(KEY_ITERATION));
+ }
+ } catch (IOException e) {
+ CLog.w("Couldn't parse the output log file");
+ CLog.e(e);
+ } catch (DeviceNotAvailableException e) {
+ CLog.w("Could not pull file: %s, error:", RESULT_DIR);
+ CLog.e(e);
+ } catch (NumberFormatException e) {
+ CLog.w("Could not find the key in file: %s, error:", KEY_NUM_ATTEMPTS);
+ CLog.e(e);
+ }
+ return postMetrics;
+ }
+ }
+
+ private ArrayList<String> getCameraIdList(String resultDir) throws DeviceNotAvailableException {
+ // The result files are created per each camera ID
+ ArrayList<String> cameraIds = new ArrayList<>();
+ IFileEntry dirEntry = getDevice().getFileEntry(resultDir);
+ if (dirEntry != null) {
+ for (IFileEntry file : dirEntry.getChildren(false)) {
+ String fileName = file.getName();
+ Matcher matcher = RESULT_FILE_REGEX.matcher(fileName);
+ if (matcher.matches()) {
+ cameraIds.add(matcher.group("id"));
+ }
+ }
+ }
+
+ if (cameraIds.isEmpty()) {
+ CLog.w("No camera ID is found in %s. The resultToFile instrumentation argument is set"
+ + " to false?", resultDir);
+ }
+ return cameraIds;
+ }
+
+ private String getResultFilePath(String cameraId) {
+ return String.format(RESULT_FILE_FORMAT, cameraId);
+ }
+}
diff --git a/src/com/android/media/tests/Camera2LatencyTest.java b/src/com/android/media/tests/Camera2LatencyTest.java
new file mode 100644
index 0000000..5a06822
--- /dev/null
+++ b/src/com/android/media/tests/Camera2LatencyTest.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Camera app latency test
+ *
+ * Runs Camera app latency test to measure Camera capture session time and reports the metrics.
+ */
+@OptionClass(alias = "camera-latency")
+public class Camera2LatencyTest extends CameraTestBase {
+
+ public Camera2LatencyTest() {
+ setTestPackage("com.google.android.camera");
+ setTestClass("com.android.camera.latency.CameraCaptureSessionTest");
+ setTestRunner("android.test.InstrumentationTestRunner");
+ setRuKey("CameraAppLatency");
+ setTestTimeoutMs(60 * 60 * 1000); // 1 hour
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ runInstrumentationTest(listener, new CollectingListener(listener));
+ }
+
+ /**
+ * A listener to collect the results from each test run, then forward them to other listeners.
+ */
+ private class CollectingListener extends DefaultCollectingListener {
+
+ public CollectingListener(ITestInvocationListener listener) {
+ super(listener);
+ }
+
+ @Override
+ public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) {
+ // Test metrics accumulated will be posted at the end of test run.
+ getAggregatedMetrics().putAll(parseResults(test.getTestName(), testMetrics));
+ }
+
+ private Map<String, String> parseResults(String testName, Map<String, String> testMetrics) {
+ final Pattern STATS_REGEX = Pattern.compile(
+ "^(?<latency>[0-9.]+)\\|(?<values>[0-9 .-]+)");
+
+ // Parse activity time stats from the instrumentation result.
+ // Format : <metric_key>=<average_of_latency>|<raw_data>
+ // Example:
+ // FirstCaptureResultTimeMs=38|13 48 ... 35
+ // SecondCaptureResultTimeMs=29.2|65 24 ... 0
+ // CreateTimeMs=373.6|382 364 ... 323
+ //
+ // Then report only the first two startup time of cold startup and average warm startup.
+ Map<String, String> parsed = new HashMap<String, String>();
+ for (Map.Entry<String, String> metric : testMetrics.entrySet()) {
+ Matcher matcher = STATS_REGEX.matcher(metric.getValue());
+ if (matcher.matches()) {
+ String keyName = String.format("%s_%s", testName, metric.getKey());
+ parsed.put(keyName, matcher.group("latency"));
+ } else {
+ CLog.w(String.format("Stats not in correct format: %s", metric.getValue()));
+ }
+ }
+ return parsed;
+ }
+ }
+}
diff --git a/src/com/android/media/tests/Camera2StressTest.java b/src/com/android/media/tests/Camera2StressTest.java
new file mode 100644
index 0000000..77868d9
--- /dev/null
+++ b/src/com/android/media/tests/Camera2StressTest.java
@@ -0,0 +1,188 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.IFileEntry;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.util.FileUtil;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Camera2 stress test
+ * Since Camera stress test can drain the battery seriously. Need to split
+ * the test suite into separate test invocation for each test method.
+ * <p/>
+ */
+@OptionClass(alias = "camera2-stress")
+public class Camera2StressTest extends CameraTestBase {
+
+ private static final String RESULT_FILE = "/sdcard/camera-out/stress.txt";
+ private static final String FAILURE_SCREENSHOT_DIR = "/sdcard/camera-screenshot/";
+ private static final String KEY_NUM_ATTEMPTS = "numAttempts";
+ private static final String KEY_ITERATION = "iteration";
+
+ public Camera2StressTest() {
+ setTestPackage("com.google.android.camera");
+ setTestClass("com.android.camera.stress.CameraStressTest");
+ setTestRunner("android.test.InstrumentationTestRunner");
+ setRuKey("CameraAppStress");
+ setTestTimeoutMs(6 * 60 * 60 * 1000); // 6 hours
+ setLogcatOnFailure(true);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ runInstrumentationTest(listener, new CollectingListener(listener));
+ }
+
+ /**
+ * A listener to collect the output from test run and fatal errors
+ */
+ private class CollectingListener extends DefaultCollectingListener {
+
+ public CollectingListener(ITestInvocationListener listener) {
+ super(listener);
+ }
+
+ @Override
+ public void testEnded(TestIdentifier test, long endTime, Map<String, String> testMetrics) {
+ if (hasTestRunFatalError()) {
+ CLog.v("The instrumentation result not found. Fall back to get the metrics from a "
+ + "log file. errorMsg: %s", getCollectingListener().getErrorMessage());
+ }
+ // TODO: Will get the additional metrics to file to prevent result loss
+
+ // Don't need to report the KEY_NUM_ATTEMPS to dashboard
+ // Iteration will be read from file
+ testMetrics.remove(KEY_NUM_ATTEMPTS);
+ testMetrics.remove(KEY_ITERATION);
+
+ // add testMethod name to the metric
+ Map<String, String> namedTestMetrics = new HashMap<>();
+ for (Entry<String, String> entry : testMetrics.entrySet()) {
+ namedTestMetrics.put(test.getTestName() + entry.getKey(), entry.getValue());
+ }
+
+ // parse the iterations metrics from the stress log files
+ parseLog(test.getTestName(), namedTestMetrics);
+
+ postScreenshotOnFailure(test);
+ super.testEnded(test, endTime, namedTestMetrics);
+ }
+
+ private void postScreenshotOnFailure(TestIdentifier test) {
+ File tmpDir = null;
+ try {
+ IFileEntry screenshotDir = getDevice().getFileEntry(FAILURE_SCREENSHOT_DIR);
+ if (screenshotDir != null && screenshotDir.isDirectory()) {
+ tmpDir = FileUtil.createTempDir("screenshot");
+ for (IFileEntry remoteFile : screenshotDir.getChildren(false)) {
+ if (remoteFile.isDirectory()) {
+ continue;
+ }
+ if (!remoteFile.getName().contains(test.getTestName())) {
+ CLog.v("Skipping the screenshot (%s) that doesn't match the current "
+ + "test (%s)", remoteFile.getName(), test.getTestName());
+ continue;
+ }
+ File screenshot = new File(tmpDir, remoteFile.getName());
+ if (!getDevice().pullFile(remoteFile.getFullPath(), screenshot)) {
+ CLog.w("Could not pull screenshot: %s", remoteFile.getFullPath());
+ continue;
+ }
+ testLog(
+ "screenshot_" + screenshot.getName(),
+ LogDataType.PNG,
+ new FileInputStreamSource(screenshot));
+ }
+ }
+ } catch (DeviceNotAvailableException e) {
+ CLog.e(e);
+ } catch (IOException e) {
+ CLog.e(e);
+ } finally {
+ FileUtil.recursiveDelete(tmpDir);
+ }
+ }
+
+ // Return null if failed to parse the result file or the test didn't even start.
+ private void parseLog(String testName, Map<String, String> testMetrics) {
+ try {
+ File outputFile = FileUtil.createTempFile("stress", ".txt");
+ getDevice().pullFile(RESULT_FILE, outputFile);
+ if (outputFile == null) {
+ throw new DeviceNotAvailableException(String.format("Failed to pull the result"
+ + "file: %s", RESULT_FILE), getDevice().getSerialNumber());
+ }
+ BufferedReader reader = new BufferedReader(new FileReader(outputFile));
+ String line;
+ Map<String, String> resultMap = new HashMap<>();
+
+ // Parse results from log file that contain the key-value pairs.
+ // eg. "numAttempts=10|iteration=9"
+ try {
+ while ((line = reader.readLine()) != null) {
+ String[] pairs = line.split("\\|");
+ for (String pair : pairs) {
+ String[] keyValue = pair.split("=");
+ // Each should be a pair of key and value.
+ String key = keyValue[0].trim();
+ String value = keyValue[1].trim();
+ resultMap.put(key, value);
+ CLog.v("%s: %s", key, value);
+ }
+ }
+ } finally {
+ reader.close();
+ }
+
+ // Fail if a stress test doesn't start.
+ if (0 == Integer.parseInt(resultMap.get(KEY_NUM_ATTEMPTS))) {
+ CLog.w("Failed to start stress tests. test setup configured incorrectly?");
+ return;
+ }
+ // Post the number of iterations only with the test name as key.
+ testMetrics.put(testName, resultMap.get(KEY_ITERATION));
+ } catch (IOException e) {
+ CLog.w("Couldn't parse the output log file:");
+ CLog.e(e);
+ } catch (DeviceNotAvailableException e) {
+ CLog.w("Could not pull file: %s, error:", RESULT_FILE);
+ CLog.e(e);
+ } catch (NumberFormatException e) {
+ CLog.w("Could not find the key in file: %s, error:", KEY_NUM_ATTEMPTS);
+ CLog.e(e);
+ }
+ }
+ }
+}
diff --git a/src/com/android/media/tests/CameraBurstStartupTest.java b/src/com/android/media/tests/CameraBurstStartupTest.java
new file mode 100644
index 0000000..74d7e87
--- /dev/null
+++ b/src/com/android/media/tests/CameraBurstStartupTest.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Camera burst startup test
+ *
+ * <p>Runs Camera device performance test to measure time for taking a burst shot.
+ */
+@OptionClass(alias = "camera-burst-shot")
+public class CameraBurstStartupTest extends CameraTestBase {
+
+ private static final Pattern STATS_REGEX = Pattern.compile("^(?<average>[0-9.]+)");
+
+ public CameraBurstStartupTest() {
+ setTestPackage("com.google.android.camera");
+ setTestClass("com.android.camera.latency.BurstStartupTest");
+ setTestRunner("android.test.InstrumentationTestRunner");
+ setRuKey("CameraBurstStartup");
+ setTestTimeoutMs(60 * 60 * 1000); // 1 hour
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ runInstrumentationTest(listener, new CollectingListener(listener));
+ }
+
+ /** A listener to collect the output from test run and fatal errors */
+ private class CollectingListener extends DefaultCollectingListener {
+
+ public CollectingListener(ITestInvocationListener listener) {
+ super(listener);
+ }
+
+ @Override
+ public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) {
+ // Test metrics accumulated will be posted at the end of test run.
+ getAggregatedMetrics().putAll(parseResults(test.getTestName(), testMetrics));
+ }
+
+ public Map<String, String> parseResults(String testName, Map<String, String> testMetrics) {
+ // Parse burst startup stats from the instrumentation result.
+ Map<String, String> parsed = new HashMap<String, String>();
+ for (Map.Entry<String, String> metric : testMetrics.entrySet()) {
+ Matcher matcher = STATS_REGEX.matcher(metric.getValue());
+
+ if (matcher.matches()) {
+ // Key name consists of a pair of test name and metric name.
+ String keyName = String.format("%s_%s", testName, metric.getKey());
+ parsed.put(keyName, matcher.group("average"));
+ } else {
+ CLog.w(String.format("Stats not in correct format: %s", metric.getValue()));
+ }
+ }
+ return parsed;
+ }
+ }
+}
diff --git a/src/com/android/media/tests/CameraLatencyTest.java b/src/com/android/media/tests/CameraLatencyTest.java
new file mode 100644
index 0000000..f44fd59
--- /dev/null
+++ b/src/com/android/media/tests/CameraLatencyTest.java
@@ -0,0 +1,369 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
+import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.CollectingTestListener;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.RegexTrie;
+import com.android.tradefed.util.StreamUtil;
+
+import junit.framework.TestCase;
+
+import org.junit.Assert;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Runs the Camera latency testcases.
+ * FIXME: more details
+ * <p/>
+ * Note that this test will not run properly unless /sdcard is mounted and writable.
+ */
+public class CameraLatencyTest implements IDeviceTest, IRemoteTest {
+ ITestDevice mTestDevice = null;
+
+ // Constants for running the tests
+ private static final String TEST_PACKAGE_NAME = "com.google.android.camera.tests";
+
+ private final String mOutputPath = "mediaStressOut.txt";
+
+ //Max timeout for the test - 30 mins
+ private static final int MAX_TEST_TIMEOUT = 30 * 60 * 1000;
+
+ /**
+ * Stores the test cases that we should consider running.
+ *
+ * <p>This currently consists of "startup" and "latency"
+ */
+ private List<TestInfo> mTestCases = new ArrayList<>();
+
+ // Options for the running the gCam test
+ @Option(name = "gCam", description = "Run gCam startup test")
+ private boolean mGcam = false;
+
+
+ /**
+ * A struct that contains useful info about the tests to run
+ */
+ static class TestInfo {
+ public String mTestName = null;
+ public String mClassName = null;
+ public String mTestMetricsName = null;
+ public RegexTrie<String> mPatternMap = new RegexTrie<>();
+
+ @Override
+ public String toString() {
+ return String.format("TestInfo: name(%s) class(%s) metric(%s) patterns(%s)", mTestName,
+ mClassName, mTestMetricsName, mPatternMap);
+ }
+ }
+
+ /**
+ * Set up the configurations for the test cases we want to run
+ */
+ private void testInfoSetup() {
+ // Startup tests
+ TestInfo t = new TestInfo();
+
+ if (mGcam) {
+ t.mTestName = "testLaunchCamera";
+ t.mClassName = "com.android.camera.stress.CameraStartUp";
+ t.mTestMetricsName = "GCameraStartup";
+ RegexTrie<String> map = t.mPatternMap;
+ map = t.mPatternMap;
+ map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)");
+ map.put("CameraStartup", "^Camera average startup time: (\\d+) ms");
+ mTestCases.add(t);
+ } else {
+ t.mTestName = "startup";
+ t.mClassName = "com.android.camera.stress.CameraStartUp";
+ t.mTestMetricsName = "CameraVideoRecorderStartup";
+ RegexTrie<String> map = t.mPatternMap;
+ map = t.mPatternMap;
+ map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)");
+ map.put("CameraStartup", "^Camera average startup time: (\\d+) ms");
+ map.put("FirstVideoStartup", "^First Video Startup: (\\d+)");
+ map.put("VideoStartup", "^Video average startup time: (\\d+) ms");
+ mTestCases.add(t);
+
+ // Latency tests
+ t = new TestInfo();
+ t.mTestName = "latency";
+ t.mClassName = "com.android.camera.stress.CameraLatency";
+ t.mTestMetricsName = "CameraLatency";
+ map = t.mPatternMap;
+ map.put("AutoFocus", "^Avg AutoFocus = (\\d+)");
+ map.put("ShutterLag", "^Avg mShutterLag = (\\d+)");
+ map.put("Preview", "^Avg mShutterToPictureDisplayedTime = (\\d+)");
+ map.put("RawPictureGeneration", "^Avg mPictureDisplayedToJpegCallbackTime = (\\d+)");
+ map.put("GenTimeDiffOverJPEGAndRaw", "^Avg mJpegCallbackFinishTime = (\\d+)");
+ map.put("FirstPreviewTime", "^Avg FirstPreviewTime = (\\d+)");
+ mTestCases.add(t);
+ }
+
+ }
+
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ Assert.assertNotNull(mTestDevice);
+ testInfoSetup();
+ for (TestInfo test : mTestCases) {
+ cleanTmpFiles();
+ executeTest(test, listener);
+ logOutputFile(test, listener);
+ }
+
+ cleanTmpFiles();
+ }
+
+ private void executeTest(TestInfo test, ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME,
+ mTestDevice.getIDevice());
+ CollectingTestListener auxListener = new CollectingTestListener();
+
+ runner.setClassName(test.mClassName);
+ runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS);
+ if (mGcam) {
+ runner.setMethodName(test.mClassName, test.mTestName);
+ }
+ mTestDevice.runInstrumentationTests(runner, listener, auxListener);
+
+ // Grab a bugreport if warranted
+ if (auxListener.hasFailedTests()) {
+ CLog.i("Grabbing bugreport after test '%s' finished with %d failures.",
+ test.mTestName, auxListener.getNumAllFailedTests());
+ InputStreamSource bugreport = mTestDevice.getBugreport();
+ listener.testLog(String.format("bugreport-%s.txt", test.mTestName),
+ LogDataType.BUGREPORT, bugreport);
+ bugreport.cancel();
+ }
+ }
+
+ /**
+ * Clean up temp files from test runs
+ * <p />
+ * Note that all photos on the test device will be removed
+ */
+ private void cleanTmpFiles() throws DeviceNotAvailableException {
+ String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+ //TODO: Remove the DCIM folder when the bug is fixed.
+ mTestDevice.executeShellCommand(String.format("rm %s/DCIM/Camera/*", extStore));
+ mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath));
+ }
+
+ /**
+ * Pull the output file from the device, add it to the logs, and also parse out the relevant
+ * test metrics and report them.
+ */
+ private void logOutputFile(TestInfo test, ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ File outputFile = null;
+ InputStreamSource outputSource = null;
+ try {
+ outputFile = mTestDevice.pullFileFromExternal(mOutputPath);
+
+ if (outputFile == null) {
+ return;
+ }
+
+ // Upload a verbatim copy of the output file
+ CLog.d("Sending %d byte file %s into the logosphere!", outputFile.length(), outputFile);
+ outputSource = new FileInputStreamSource(outputFile);
+ listener.testLog(String.format("output-%s.txt", test.mTestName), LogDataType.TEXT,
+ outputSource);
+
+ // Parse the output file to upload aggregated metrics
+ parseOutputFile(test, new FileInputStream(outputFile), listener);
+ } catch (IOException e) {
+ CLog.e("IOException while reading or parsing output file");
+ CLog.e(e);
+ } finally {
+ FileUtil.deleteFile(outputFile);
+ StreamUtil.cancel(outputSource);
+ }
+ }
+
+ /**
+ * Parse the relevant metrics from the Instrumentation test output file
+ */
+ private void parseOutputFile(TestInfo test, InputStream dataStream,
+ ITestInvocationListener listener) {
+ Map<String, String> runMetrics = new HashMap<>();
+
+ // try to parse it
+ String contents;
+ try {
+ contents = StreamUtil.getStringFromStream(dataStream);
+ } catch (IOException e) {
+ CLog.e("Got IOException during %s test processing", test.mTestName);
+ CLog.e(e);
+ return;
+ }
+
+ List<String> lines = Arrays.asList(contents.split("\n"));
+ ListIterator<String> lineIter = lines.listIterator();
+ String line;
+ while (lineIter.hasNext()) {
+ line = lineIter.next();
+ List<List<String>> capture = new ArrayList<>(1);
+ String key = test.mPatternMap.retrieve(capture, line);
+ if (key != null) {
+ CLog.d("Got %s key '%s' and captures '%s'", test.mTestName, key,
+ capture.toString());
+ } else if (line.isEmpty()) {
+ // ignore
+ continue;
+ } else {
+ CLog.d("Got unmatched line: %s", line);
+ continue;
+ }
+
+ runMetrics.put(key, capture.get(0).get(0));
+ }
+
+ reportMetrics(listener, test, runMetrics);
+ }
+
+ /**
+ * Report run metrics by creating an empty test run to stick them in
+ * <p />
+ * Exposed for unit testing
+ */
+ void reportMetrics(ITestInvocationListener listener, TestInfo test,
+ Map<String, String> metrics) {
+ // Create an empty testRun to report the parsed runMetrics
+ CLog.d("About to report metrics for %s: %s", test.mTestMetricsName, metrics);
+ listener.testRunStarted(test.mTestMetricsName, 0);
+ listener.testRunEnded(0, metrics);
+ }
+
+ @Override
+ public void setDevice(ITestDevice device) {
+ mTestDevice = device;
+ }
+
+ @Override
+ public ITestDevice getDevice() {
+ return mTestDevice;
+ }
+
+ /**
+ * A meta-test to ensure that bits of the CameraLatencyTest are working properly
+ */
+ public static class MetaTest extends TestCase {
+ private CameraLatencyTest mTestInstance = null;
+
+ private TestInfo mTestInfo = null;
+
+ private TestInfo mReportedTestInfo = null;
+ private Map<String, String> mReportedMetrics = null;
+
+ private static String join(String... pieces) {
+ StringBuilder sb = new StringBuilder();
+ for (String piece : pieces) {
+ sb.append(piece);
+ sb.append("\n");
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ mTestInstance = new CameraLatencyTest() {
+ @Override
+ void reportMetrics(ITestInvocationListener l, TestInfo test,
+ Map<String, String> metrics) {
+ mReportedTestInfo = test;
+ mReportedMetrics = metrics;
+ }
+ };
+
+ // Startup tests
+ mTestInfo = new TestInfo();
+ TestInfo t = mTestInfo; // convenience alias
+ t.mTestName = "startup";
+ t.mClassName = "com.android.camera.stress.CameraStartUp";
+ t.mTestMetricsName = "camera_video_recorder_startup";
+ RegexTrie<String> map = t.mPatternMap;
+ map.put("FirstCameraStartup", "^First Camera Startup: (\\d+)");
+ map.put("CameraStartup", "^Camera average startup time: (\\d+) ms");
+ map.put("FirstVideoStartup", "^First Video Startup: (\\d+)");
+ map.put("VideoStartup", "^Video average startup time: (\\d+) ms");
+ map.put("FirstPreviewTime", "^Avg FirstPreviewTime = (\\d+)");
+ }
+
+ /**
+ * Make sure that parsing works in the expected case
+ */
+ public void testParse() throws Exception {
+ String output = join(
+ "First Camera Startup: 1421", /* "FirstCameraStartup" key */
+ "Camerastartup time: ",
+ "Number of loop: 19",
+ "Individual Camera Startup Time = 1920 ,1929 ,1924 ,1871 ,1840 ,1892 ,1813 " +
+ ",1927 ,1856 ,1929 ,1911 ,1873 ,1381 ,1888 ,2405 ,1926 ,1840 ,2502 " +
+ ",2357 ,",
+ "",
+ "Camera average startup time: 1946 ms", /* "CameraStartup" key */
+ "",
+ "First Video Startup: 2176", /* "FirstVideoStartup" key */
+ "Camera Latency : ",
+ "Number of loop: 20",
+ "Avg AutoFocus = 2304",
+ "Avg mShutterLag = 403",
+ "Avg mShutterToPictureDisplayedTime = 369",
+ "Avg mPictureDisplayedToJpegCallbackTime = 50",
+ "Avg mJpegCallbackFinishTime = 1679",
+ "Avg FirstPreviewTime = 1340");
+
+ InputStream iStream = new ByteArrayInputStream(output.getBytes());
+ mTestInstance.parseOutputFile(mTestInfo, iStream, null);
+ assertEquals(mTestInfo, mReportedTestInfo);
+ assertNotNull(mReportedMetrics);
+ assertEquals(4, mReportedMetrics.size());
+ assertEquals("1946", mReportedMetrics.get("CameraStartup"));
+ assertEquals("2176", mReportedMetrics.get("FirstVideoStartup"));
+ assertEquals("1421", mReportedMetrics.get("FirstCameraStartup"));
+ assertEquals("1340", mReportedMetrics.get("FirstPreviewTime"));
+ }
+ }
+}
diff --git a/src/com/android/media/tests/CameraPerformanceTest.java b/src/com/android/media/tests/CameraPerformanceTest.java
new file mode 100644
index 0000000..ebaad68
--- /dev/null
+++ b/src/com/android/media/tests/CameraPerformanceTest.java
@@ -0,0 +1,672 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.util.FileUtil;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.xmlpull.v1.XmlPullParser;
+import org.xmlpull.v1.XmlPullParserException;
+import org.xmlpull.v1.XmlPullParserFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * This test invocation runs android.hardware.camera2.cts.PerformanceTest - Camera2 API use case
+ * performance KPIs (Key Performance Indicator), such as camera open time, session creation time,
+ * shutter lag etc. The KPI data will be parsed and reported.
+ */
+@OptionClass(alias = "camera-framework")
+public class CameraPerformanceTest extends CameraTestBase {
+
+ private static final String TEST_CAMERA_LAUNCH = "testCameraLaunch";
+ private static final String TEST_SINGLE_CAPTURE = "testSingleCapture";
+ private static final String TEST_REPROCESSING_LATENCY = "testReprocessingLatency";
+ private static final String TEST_REPROCESSING_THROUGHPUT = "testReprocessingThroughput";
+
+ // KPIs to be reported. The key is test methods and the value is KPIs in the method.
+ private final ImmutableMultimap<String, String> mReportingKpis =
+ new ImmutableMultimap.Builder<String, String>()
+ .put(TEST_CAMERA_LAUNCH, "Camera launch time")
+ .put(TEST_CAMERA_LAUNCH, "Camera start preview time")
+ .put(TEST_SINGLE_CAPTURE, "Camera capture result latency")
+ .put(TEST_REPROCESSING_LATENCY, "YUV reprocessing shot to shot latency")
+ .put(TEST_REPROCESSING_LATENCY, "opaque reprocessing shot to shot latency")
+ .put(TEST_REPROCESSING_THROUGHPUT, "YUV reprocessing capture latency")
+ .put(TEST_REPROCESSING_THROUGHPUT, "opaque reprocessing capture latency")
+ .build();
+
+ // JSON format keymap, key is test method name and the value is stream name in Json file
+ private static final ImmutableMap<String, String> METHOD_JSON_KEY_MAP =
+ new ImmutableMap.Builder<String, String>()
+ .put(TEST_CAMERA_LAUNCH, "test_camera_launch")
+ .put(TEST_SINGLE_CAPTURE, "test_single_capture")
+ .put(TEST_REPROCESSING_LATENCY, "test_reprocessing_latency")
+ .put(TEST_REPROCESSING_THROUGHPUT, "test_reprocessing_throughput")
+ .build();
+
+ private <E extends Number> double getAverage(List<E> list) {
+ double sum = 0;
+ int size = list.size();
+ for (E num : list) {
+ sum += num.doubleValue();
+ }
+ if (size == 0) {
+ return 0.0;
+ }
+ return (sum / size);
+ }
+
+ public CameraPerformanceTest() {
+ // Set up the default test info. But this is subject to be overwritten by options passed
+ // from commands.
+ setTestPackage("android.camera.cts");
+ setTestClass("android.hardware.camera2.cts.PerformanceTest");
+ setTestRunner("android.support.test.runner.AndroidJUnitRunner");
+ setRuKey("camera_framework_performance");
+ setTestTimeoutMs(10 * 60 * 1000); // 10 mins
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ runInstrumentationTest(listener, new CollectingListener(listener));
+ }
+
+ /**
+ * A listener to collect the output from test run and fatal errors
+ */
+ private class CollectingListener extends DefaultCollectingListener {
+
+ public CollectingListener(ITestInvocationListener listener) {
+ super(listener);
+ }
+
+ @Override
+ public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) {
+ // Pass the test name for a key in the aggregated metrics, because
+ // it is used to generate the key of the final metrics to post at the end of test run.
+ for (Map.Entry<String, String> metric : testMetrics.entrySet()) {
+ getAggregatedMetrics().put(test.getTestName(), metric.getValue());
+ }
+ }
+
+ @Override
+ public void handleTestRunEnded(
+ ITestInvocationListener listener,
+ long elapsedTime,
+ Map<String, String> runMetrics) {
+ // Report metrics at the end of test run.
+ Map<String, String> result = parseResult(getAggregatedMetrics());
+ listener.testRunEnded(getTestDurationMs(), result);
+ }
+ }
+
+ /**
+ * Parse Camera Performance KPIs results and then put them all together to post the final
+ * report.
+ *
+ * @return a {@link HashMap} that contains pairs of kpiName and kpiValue
+ */
+ private Map<String, String> parseResult(Map<String, String> metrics) {
+
+ // if json report exists, return the parse results
+ CtsJsonResultParser ctsJsonResultParser = new CtsJsonResultParser();
+
+ if (ctsJsonResultParser.isJsonFileExist()) {
+ return ctsJsonResultParser.parse();
+ }
+
+ Map<String, String> resultsAll = new HashMap<String, String>();
+
+ CtsResultParserBase parser;
+ for (Map.Entry<String, String> metric : metrics.entrySet()) {
+ String testMethod = metric.getKey();
+ String testResult = metric.getValue();
+ CLog.d("test name %s", testMethod);
+ CLog.d("test result %s", testResult);
+ // Probe which result parser should be used.
+ if (shouldUseCtsXmlResultParser(testResult)) {
+ parser = new CtsXmlResultParser();
+ } else {
+ parser = new CtsDelimitedResultParser();
+ }
+
+ // Get pairs of { KPI name, KPI value } from stdout that each test outputs.
+ // Assuming that a device has both the front and back cameras, parser will return
+ // 2 KPIs in HashMap. For an example of testCameraLaunch,
+ // {
+ // ("Camera 0 Camera launch time", "379.2"),
+ // ("Camera 1 Camera launch time", "272.8"),
+ // }
+ Map<String, String> testKpis = parser.parse(testResult, testMethod);
+ for (String k : testKpis.keySet()) {
+ if (resultsAll.containsKey(k)) {
+ throw new RuntimeException(
+ String.format("KPI name (%s) conflicts with the existing names.", k));
+ }
+ }
+ parser.clear();
+
+ // Put each result together to post the final result
+ resultsAll.putAll(testKpis);
+ }
+ return resultsAll;
+ }
+
+ public boolean shouldUseCtsXmlResultParser(String result) {
+ final String XML_DECLARATION = "<?xml";
+ return (result.startsWith(XML_DECLARATION)
+ || result.startsWith(XML_DECLARATION.toUpperCase()));
+ }
+
+ /** Data class of CTS test results for Camera framework performance test */
+ public static class CtsMetric {
+ String testMethod; // "testSingleCapture"
+ String source; // "android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327"
+ // or "testSingleCapture" (just test method name)
+ String message; // "Camera 0: Camera capture latency"
+ String type; // "lower_better"
+ String unit; // "ms"
+ String value; // "691.0" (is an average of 736.0 688.0 679.0 667.0 686.0)
+ String schemaKey; // RU schema key = message (+ testMethodName if needed), derived
+
+ // eg. "android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327"
+ public static final Pattern SOURCE_REGEX =
+ Pattern.compile("^(?<package>[a-zA-Z\\d\\._$]+)#(?<method>[a-zA-Z\\d_$]+)(:\\d+)?");
+ // eg. "Camera 0: Camera capture latency"
+ public static final Pattern MESSAGE_REGEX =
+ Pattern.compile("^Camera\\s+(?<cameraId>\\d+):\\s+(?<kpiName>.*)");
+
+ CtsMetric(
+ String testMethod,
+ String source,
+ String message,
+ String type,
+ String unit,
+ String value) {
+ this.testMethod = testMethod;
+ this.source = source;
+ this.message = message;
+ this.type = type;
+ this.unit = unit;
+ this.value = value;
+ this.schemaKey = getRuSchemaKeyName(message);
+ }
+
+ public boolean matches(String testMethod, String kpiName) {
+ return (this.testMethod.equals(testMethod) && this.message.endsWith(kpiName));
+ }
+
+ public String getRuSchemaKeyName(String message) {
+ // Note 1: The key shouldn't contain ":" for side by side report.
+ String schemaKey = message.replace(":", "");
+ // Note 2: Two tests testReprocessingLatency & testReprocessingThroughput have the
+ // same metric names to report results. To make the report key name distinct,
+ // the test name is added as prefix for these tests for them.
+ final String[] TEST_NAMES_AS_PREFIX = {
+ "testReprocessingLatency", "testReprocessingThroughput"
+ };
+ for (String testName : TEST_NAMES_AS_PREFIX) {
+ if (testMethod.endsWith(testName)) {
+ schemaKey = String.format("%s_%s", testName, schemaKey);
+ break;
+ }
+ }
+ return schemaKey;
+ }
+
+ public String getTestMethodNameInSource(String source) {
+ Matcher m = SOURCE_REGEX.matcher(source);
+ if (!m.matches()) {
+ return source;
+ }
+ return m.group("method");
+ }
+ }
+
+ /**
+ * Base class of CTS test result parser. This is inherited to two derived parsers,
+ * {@link CtsDelimitedResultParser} for legacy delimiter separated format and
+ * {@link CtsXmlResultParser} for XML typed format introduced since NYC.
+ */
+ public abstract class CtsResultParserBase {
+
+ protected CtsMetric mSummary;
+ protected List<CtsMetric> mDetails = new ArrayList<>();
+
+ /**
+ * Parse Camera Performance KPIs result first, then leave the only KPIs that matter.
+ *
+ * @param result String to be parsed
+ * @param testMethod test method name used to leave the only metric that matters
+ * @return a {@link HashMap} that contains kpiName and kpiValue
+ */
+ public abstract Map<String, String> parse(String result, String testMethod);
+
+ protected Map<String, String> filter(List<CtsMetric> metrics, String testMethod) {
+ Map<String, String> filtered = new HashMap<String, String>();
+ for (CtsMetric metric : metrics) {
+ for (String kpiName : mReportingKpis.get(testMethod)) {
+ // Post the data only when it matches with the given methods and KPI names.
+ if (metric.matches(testMethod, kpiName)) {
+ filtered.put(metric.schemaKey, metric.value);
+ }
+ }
+ }
+ return filtered;
+ }
+
+ protected void setSummary(CtsMetric summary) {
+ mSummary = summary;
+ }
+
+ protected void addDetail(CtsMetric detail) {
+ mDetails.add(detail);
+ }
+
+ protected List<CtsMetric> getDetails() {
+ return mDetails;
+ }
+
+ void clear() {
+ mSummary = null;
+ mDetails.clear();
+ }
+ }
+
+ /**
+ * Parses the camera performance test generated by the underlying instrumentation test and
+ * returns it to test runner for later reporting.
+ *
+ * <p>TODO(liuyg): Rename this class to not reference CTS.
+ *
+ * <p>Format: (summary message)| |(type)|(unit)|(value) ++++
+ * (source)|(message)|(type)|(unit)|(value)... +++ ...
+ *
+ * <p>Example: Camera launch average time for Camera 1| |lower_better|ms|586.6++++
+ * android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171| Camera 0: Camera open
+ * time|lower_better|ms|74.0 100.0 70.0 67.0 82.0 +++
+ * android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171| Camera 0: Camera configure
+ * stream time|lower_better|ms|9.0 5.0 5.0 8.0 5.0 ...
+ *
+ * <p>See also com.android.cts.util.ReportLog for the format detail.
+ */
+ public class CtsDelimitedResultParser extends CtsResultParserBase {
+ private static final String LOG_SEPARATOR = "\\+\\+\\+";
+ private static final String SUMMARY_SEPARATOR = "\\+\\+\\+\\+";
+ private final Pattern mSummaryRegex =
+ Pattern.compile(
+ "^(?<message>[^|]+)\\| \\|(?<type>[^|]+)\\|(?<unit>[^|]+)\\|(?<value>[0-9 .]+)");
+ private final Pattern mDetailRegex =
+ Pattern.compile(
+ "^(?<source>[^|]+)\\|(?<message>[^|]+)\\|(?<type>[^|]+)\\|(?<unit>[^|]+)\\|"
+ + "(?<values>[0-9 .]+)");
+
+ @Override
+ public Map<String, String> parse(String result, String testMethod) {
+ parseToCtsMetrics(result, testMethod);
+ parseToCtsMetrics(result, testMethod);
+ return filter(getDetails(), testMethod);
+ }
+
+ void parseToCtsMetrics(String result, String testMethod) {
+ // Split summary and KPIs from stdout passes as parameter.
+ String[] output = result.split(SUMMARY_SEPARATOR);
+ if (output.length != 2) {
+ throw new RuntimeException("Value not in the correct format");
+ }
+ Matcher summaryMatcher = mSummaryRegex.matcher(output[0].trim());
+
+ // Parse summary.
+ // Example: "Camera launch average time for Camera 1| |lower_better|ms|586.6++++"
+ if (summaryMatcher.matches()) {
+ setSummary(
+ new CtsMetric(
+ testMethod,
+ null,
+ summaryMatcher.group("message"),
+ summaryMatcher.group("type"),
+ summaryMatcher.group("unit"),
+ summaryMatcher.group("value")));
+ } else {
+ // Fall through since the summary is not posted as results.
+ CLog.w("Summary not in the correct format");
+ }
+
+ // Parse KPIs.
+ // Example: "android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171|Camera 0:
+ // Camera open time|lower_better|ms|74.0 100.0 70.0 67.0 82.0 +++"
+ String[] details = output[1].split(LOG_SEPARATOR);
+ for (String detail : details) {
+ Matcher detailMatcher = mDetailRegex.matcher(detail.trim());
+ if (detailMatcher.matches()) {
+ // get average of kpi values
+ List<Double> values = new ArrayList<>();
+ for (String value : detailMatcher.group("values").split("\\s+")) {
+ values.add(Double.parseDouble(value));
+ }
+ String kpiValue = String.format("%.1f", getAverage(values));
+ addDetail(
+ new CtsMetric(
+ testMethod,
+ detailMatcher.group("source"),
+ detailMatcher.group("message"),
+ detailMatcher.group("type"),
+ detailMatcher.group("unit"),
+ kpiValue));
+ } else {
+ throw new RuntimeException("KPI not in the correct format");
+ }
+ }
+ }
+ }
+
+ /**
+ * Parses the CTS test results in a XML format introduced since NYC.
+ * Format:
+ * <Summary>
+ * <Metric source="android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327"
+ * message="Camera capture result average latency for all cameras "
+ * score_type="lower_better"
+ * score_unit="ms"
+ * <Value>353.9</Value>
+ * </Metric>
+ * </Summary>
+ * <Detail>
+ * <Metric source="android.hardware.camera2.cts.PerformanceTest#testSingleCapture:303"
+ * message="Camera 0: Camera capture latency"
+ * score_type="lower_better"
+ * score_unit="ms">
+ * <Value>335.0</Value>
+ * <Value>302.0</Value>
+ * <Value>316.0</Value>
+ * </Metric>
+ * </Detail>
+ * }
+ * See also com.android.compatibility.common.util.ReportLog for the format detail.
+ */
+ public class CtsXmlResultParser extends CtsResultParserBase {
+ private static final String ENCODING = "UTF-8";
+ // XML constants
+ private static final String DETAIL_TAG = "Detail";
+ private static final String METRIC_TAG = "Metric";
+ private static final String MESSAGE_ATTR = "message";
+ private static final String SCORETYPE_ATTR = "score_type";
+ private static final String SCOREUNIT_ATTR = "score_unit";
+ private static final String SOURCE_ATTR = "source";
+ private static final String SUMMARY_TAG = "Summary";
+ private static final String VALUE_TAG = "Value";
+ private String mTestMethod;
+
+ @Override
+ public Map<String, String> parse(String result, String testMethod) {
+ try {
+ mTestMethod = testMethod;
+ XmlPullParserFactory factory = XmlPullParserFactory.newInstance();
+ XmlPullParser parser = factory.newPullParser();
+ parser.setInput(new ByteArrayInputStream(result.getBytes(ENCODING)), ENCODING);
+ parser.nextTag();
+ parse(parser);
+ return filter(getDetails(), testMethod);
+ } catch (XmlPullParserException | IOException e) {
+ throw new RuntimeException("Failed to parse results in XML.", e);
+ }
+ }
+
+ /**
+ * Parses a {@link CtsMetric} from the given XML parser.
+ *
+ * @param parser
+ * @throws IOException
+ * @throws XmlPullParserException
+ */
+ private void parse(XmlPullParser parser) throws XmlPullParserException, IOException {
+ parser.require(XmlPullParser.START_TAG, null, SUMMARY_TAG);
+ parser.nextTag();
+ setSummary(parseToCtsMetrics(parser));
+ parser.nextTag();
+ parser.require(XmlPullParser.END_TAG, null, SUMMARY_TAG);
+ parser.nextTag();
+ if (parser.getName().equals(DETAIL_TAG)) {
+ while (parser.nextTag() == XmlPullParser.START_TAG) {
+ addDetail(parseToCtsMetrics(parser));
+ }
+ parser.require(XmlPullParser.END_TAG, null, DETAIL_TAG);
+ }
+ }
+
+ CtsMetric parseToCtsMetrics(XmlPullParser parser)
+ throws IOException, XmlPullParserException {
+ parser.require(XmlPullParser.START_TAG, null, METRIC_TAG);
+ String source = parser.getAttributeValue(null, SOURCE_ATTR);
+ String message = parser.getAttributeValue(null, MESSAGE_ATTR);
+ String type = parser.getAttributeValue(null, SCORETYPE_ATTR);
+ String unit = parser.getAttributeValue(null, SCOREUNIT_ATTR);
+ List<Double> values = new ArrayList<>();
+ while (parser.nextTag() == XmlPullParser.START_TAG) {
+ parser.require(XmlPullParser.START_TAG, null, VALUE_TAG);
+ values.add(Double.parseDouble(parser.nextText()));
+ parser.require(XmlPullParser.END_TAG, null, VALUE_TAG);
+ }
+ String kpiValue = String.format("%.1f", getAverage(values));
+ parser.require(XmlPullParser.END_TAG, null, METRIC_TAG);
+ return new CtsMetric(mTestMethod, source, message, type, unit, kpiValue);
+ }
+ }
+
+ /*
+ * Parse the Json report from the Json String
+ * "test_single_capture":
+ * {"camera_id":"0","camera_capture_latency":[264.0,229.0,229.0,237.0,234.0],
+ * "camera_capture_result_latency":[230.0,197.0,196.0,204.0,202.0]},"
+ * "test_reprocessing_latency":
+ * {"camera_id":"0","format":35,"reprocess_type":"YUV reprocessing",
+ * "capture_message":"shot to shot latency","latency":[102.0,101.0,99.0,99.0,100.0,101.0],
+ * "camera_reprocessing_shot_to_shot_average_latency":100.33333333333333},
+ *
+ * TODO: move this to a seperate class
+ */
+ public class CtsJsonResultParser {
+
+ // report json file set in
+ // cts/tools/cts-tradefed/res/config/cts-preconditions.xml
+ private static final String JSON_RESULT_FILE =
+ "/sdcard/report-log-files/CtsCameraTestCases.reportlog.json";
+ private static final String CAMERA_ID_KEY = "camera_id";
+ private static final String AVERAGE_LATENCY_KEY = "average_latency";
+ private static final String REPROCESS_TYPE_KEY = "reprocess_type";
+ private static final String CAPTURE_MESSAGE_KEY = "capture_message";
+ private static final String LATENCY_KEY = "latency";
+
+ public Map<String, String> parse() {
+
+ Map<String, String> metrics = new HashMap<>();
+
+ String jsonString = getFormatedJsonReportFromFile();
+ if (null == jsonString) {
+ throw new RuntimeException("Get null json report string.");
+ }
+
+ Map<String, List<Double>> metricsData = new HashMap<>();
+
+ try {
+ JSONObject jsonObject = new JSONObject(jsonString);
+
+ for (String testMethod : METHOD_JSON_KEY_MAP.keySet()) {
+
+ JSONArray jsonArray =
+ (JSONArray) jsonObject.get(METHOD_JSON_KEY_MAP.get(testMethod));
+
+ switch (testMethod) {
+ case TEST_REPROCESSING_THROUGHPUT:
+ case TEST_REPROCESSING_LATENCY:
+ for (int i = 0; i < jsonArray.length(); i++) {
+ JSONObject element = jsonArray.getJSONObject(i);
+
+ // create a kpiKey from camera id,
+ // reprocess type and capture message
+ String cameraId = element.getString(CAMERA_ID_KEY);
+ String reprocessType = element.getString(REPROCESS_TYPE_KEY);
+ String captureMessage = element.getString(CAPTURE_MESSAGE_KEY);
+ String kpiKey =
+ String.format(
+ "%s_Camera %s %s %s",
+ testMethod,
+ cameraId,
+ reprocessType,
+ captureMessage);
+
+ // read the data array from json object
+ JSONArray jsonDataArray = element.getJSONArray(LATENCY_KEY);
+ if (!metricsData.containsKey(kpiKey)) {
+ List<Double> list = new ArrayList<>();
+ metricsData.put(kpiKey, list);
+ }
+ for (int j = 0; j < jsonDataArray.length(); j++) {
+ metricsData.get(kpiKey).add(jsonDataArray.getDouble(j));
+ }
+ }
+ break;
+ case TEST_SINGLE_CAPTURE:
+ case TEST_CAMERA_LAUNCH:
+ for (int i = 0; i < jsonArray.length(); i++) {
+ JSONObject element = jsonArray.getJSONObject(i);
+
+ String cameraid = element.getString(CAMERA_ID_KEY);
+ for (String kpiName : mReportingKpis.get(testMethod)) {
+
+ // the json key is all lower case
+ String jsonKey = kpiName.toLowerCase().replace(" ", "_");
+ String kpiKey =
+ String.format("Camera %s %s", cameraid, kpiName);
+ if (!metricsData.containsKey(kpiKey)) {
+ List<Double> list = new ArrayList<>();
+ metricsData.put(kpiKey, list);
+ }
+ JSONArray jsonDataArray = element.getJSONArray(jsonKey);
+ for (int j = 0; j < jsonDataArray.length(); j++) {
+ metricsData.get(kpiKey).add(jsonDataArray.getDouble(j));
+ }
+ }
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ } catch (JSONException e) {
+ CLog.w("JSONException: %s in string %s", e.getMessage(), jsonString);
+ }
+
+ // take the average of all data for reporting
+ for (String kpiKey : metricsData.keySet()) {
+ String kpiValue = String.format("%.1f", getAverage(metricsData.get(kpiKey)));
+ metrics.put(kpiKey, kpiValue);
+ }
+ return metrics;
+ }
+
+ public boolean isJsonFileExist() {
+ try {
+ return getDevice().doesFileExist(JSON_RESULT_FILE);
+ } catch (DeviceNotAvailableException e) {
+ throw new RuntimeException("Failed to check json report file on device.", e);
+ }
+ }
+
+ /*
+ * read json report file on the device
+ */
+ private String getFormatedJsonReportFromFile() {
+ String jsonString = null;
+ try {
+ // pull the json report file from device
+ File outputFile = FileUtil.createTempFile("json", ".txt");
+ getDevice().pullFile(JSON_RESULT_FILE, outputFile);
+ jsonString = reformatJsonString(FileUtil.readStringFromFile(outputFile));
+ } catch (IOException e) {
+ CLog.w("Couldn't parse the output json log file: ", e);
+ } catch (DeviceNotAvailableException e) {
+ CLog.w("Could not pull file: %s, error: %s", JSON_RESULT_FILE, e);
+ }
+ return jsonString;
+ }
+
+ // Reformat the json file to remove duplicate keys
+ private String reformatJsonString(String jsonString) {
+
+ final String TEST_METRICS_PATTERN = "\\\"([a-z0-9_]*)\\\":(\\{[^{}]*\\})";
+ StringBuilder newJsonBuilder = new StringBuilder();
+ // Create map of stream names and json objects.
+ HashMap<String, List<String>> jsonMap = new HashMap<>();
+ Pattern p = Pattern.compile(TEST_METRICS_PATTERN);
+ Matcher m = p.matcher(jsonString);
+ while (m.find()) {
+ String key = m.group(1);
+ String value = m.group(2);
+ if (!jsonMap.containsKey(key)) {
+ jsonMap.put(key, new ArrayList<String>());
+ }
+ jsonMap.get(key).add(value);
+ }
+ // Rewrite json string as arrays.
+ newJsonBuilder.append("{");
+ boolean firstLine = true;
+ for (String key : jsonMap.keySet()) {
+ if (!firstLine) {
+ newJsonBuilder.append(",");
+ } else {
+ firstLine = false;
+ }
+ newJsonBuilder.append("\"").append(key).append("\":[");
+ boolean firstValue = true;
+ for (String stream : jsonMap.get(key)) {
+ if (!firstValue) {
+ newJsonBuilder.append(",");
+ } else {
+ firstValue = false;
+ }
+ newJsonBuilder.append(stream);
+ }
+ newJsonBuilder.append("]");
+ }
+ newJsonBuilder.append("}");
+ return newJsonBuilder.toString();
+ }
+ }
+}
diff --git a/src/com/android/media/tests/CameraSettingsTest.java b/src/com/android/media/tests/CameraSettingsTest.java
new file mode 100644
index 0000000..d5fe0dd
--- /dev/null
+++ b/src/com/android/media/tests/CameraSettingsTest.java
@@ -0,0 +1,222 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
+import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.BugreportCollector;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.StreamUtil;
+
+import org.junit.Assert;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Camera zoom stress test that increments the camera's zoom level across the
+ * entire range [min, max], taking a picture at each level.
+ */
+public class CameraSettingsTest implements IDeviceTest, IRemoteTest {
+
+ private static final String ZOOM_STANZA = "testStressCameraZoom";
+ private static final String SCENE_MODES_STANZA = "testStressCameraSceneModes";
+ private static final Pattern EXPECTED_LOOP_COUNT_PATTERN =
+ Pattern.compile("(Total number of loops:)(\\s*)(\\d+)");
+ private static final Pattern ACTUAL_LOOP_COUNT_PATTERN =
+ Pattern.compile("(No of loop:)(.*,\\s)(\\d+)$");
+
+ private static final String TEST_CLASS_NAME =
+ "com.android.mediaframeworktest.stress.CameraStressTest";
+ private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest";
+ private static final String TEST_RUNNER_NAME =
+ "com.android.mediaframeworktest.CameraStressTestRunner";
+ private static final String TEST_RU = "CameraApplicationStress";
+
+ private final String mOutputPath = "cameraStressOutput.txt";
+ private static final int MAX_TIME_OUT = 90 * 60 * 1000; //90 mins
+
+ @Option(name="testMethodName", description="Used to specify a specific test method to run")
+ private String mTestMethodName = null;
+
+ ITestDevice mTestDevice = null;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ Assert.assertNotNull(mTestDevice);
+
+ IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME,
+ TEST_RUNNER_NAME, mTestDevice.getIDevice());
+ runner.setClassName(TEST_CLASS_NAME);
+
+ if (mTestMethodName != null) {
+ runner.setMethodName(TEST_CLASS_NAME, mTestMethodName);
+ }
+ runner.setMaxTimeToOutputResponse(MAX_TIME_OUT, TimeUnit.MILLISECONDS);
+
+ BugreportCollector bugListener = new BugreportCollector(listener, mTestDevice);
+ bugListener.addPredicate(BugreportCollector.AFTER_FAILED_TESTCASES);
+ bugListener.setDescriptiveName(this.getClass().getName());
+ Assert.assertTrue(mTestDevice.runInstrumentationTests(runner, bugListener));
+
+ Map<String, String> metrics = parseOutputFile();
+ reportMetrics(bugListener, TEST_RU, metrics);
+ cleanupDevice();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mTestDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ITestDevice getDevice() {
+ return mTestDevice;
+ }
+
+ /**
+ * Wipes the device's external memory of test collateral from prior runs.
+ *
+ * @throws DeviceNotAvailableException If the device is unavailable or
+ * something happened while deleting files
+ */
+ private void cleanupDevice() throws DeviceNotAvailableException {
+ String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+ mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath));
+ }
+
+ /**
+ * Parses the output file generated by the underlying instrumentation test
+ * and returns it to the main driver for later reporting.
+ *
+ * @return The {@link Map} that contains metrics for the test.
+ * @throws DeviceNotAvailableException If the device is unavailable or
+ * something happened while deleting files
+ */
+ private Map<String, String> parseOutputFile() throws DeviceNotAvailableException {
+ File outputFile = null;
+ BufferedReader reader = null;
+ ArrayList<String> lines = new ArrayList<String>();
+ String line = null;
+ String key = null;
+ Integer expectedCount = null;
+ Integer actualCount = null;
+ ListIterator<String> listIterator = null;
+ Map<String, String> metrics = new HashMap<String, String>();
+
+ // Read in data
+ try {
+ outputFile = mTestDevice.pullFileFromExternal(mOutputPath);
+ reader = new BufferedReader(new FileReader(outputFile));
+
+ while ((line = reader.readLine()) != null) {
+ if (!line.isEmpty()) {
+ lines.add(line);
+ }
+ }
+ } catch (IOException e) {
+ CLog.e(String.format("IOException reading from file: %s", e.toString()));
+ } finally {
+ StreamUtil.close(reader);
+ }
+
+ // Output file looks like:
+ // Test name:
+ // Total number of loops: 123
+ // No of loop: 0, 1, 2, 3, ..., 122 (0 based)
+ // Note that the actual count should be +1 as the # of loop is 0 based.
+ listIterator = lines.listIterator();
+
+ while (listIterator.hasNext()) {
+ line = listIterator.next();
+ CLog.d(String.format("Parsing line: \"%s\"", line));
+
+ if (ZOOM_STANZA.equals(line)) {
+ key = "CameraZoom";
+ } else if (SCENE_MODES_STANZA.equals(line)) {
+ key = "CameraSceneMode";
+ }
+
+ Matcher expectedMatcher = EXPECTED_LOOP_COUNT_PATTERN.matcher(line);
+ if (expectedMatcher.matches()) {
+ expectedCount = Integer.valueOf(expectedMatcher.group(3));
+ CLog.d(String.format("Found expected count for key \"%s\": %s",
+ key, expectedCount));
+ }
+
+ Matcher actualMatcher = ACTUAL_LOOP_COUNT_PATTERN.matcher(line);
+ if (actualMatcher.matches()) {
+ actualCount = 1 + Integer.valueOf(actualMatcher.group(3));
+ CLog.d(String.format("Found actual count for key \"%s\": %s", key, actualCount));
+ }
+
+ if ((key != null) && (expectedCount != null) && (actualCount != null)) {
+ metrics.put(key, String.format("%d", actualCount));
+ key = null;
+ expectedCount = null;
+ actualCount = null;
+ }
+ }
+
+ return metrics;
+ }
+
+ /**
+ * Report run metrics by creating an empty test run to stick them in.
+ *
+ * @param listener The {@link ITestInvocationListener} of test results
+ * @param runName The test name
+ * @param metrics The {@link Map} that contains metrics for the given test
+ */
+ private void reportMetrics(ITestInvocationListener listener, String runName,
+ Map<String, String> metrics) {
+ InputStreamSource bugreport = mTestDevice.getBugreport();
+ listener.testLog("bugreport", LogDataType.BUGREPORT, bugreport);
+ bugreport.cancel();
+
+ CLog.d(String.format("About to report metrics: %s", metrics));
+ listener.testRunStarted(runName, 0);
+ listener.testRunEnded(0, metrics);
+ }
+}
diff --git a/src/com/android/media/tests/CameraShotLatencyTest.java b/src/com/android/media/tests/CameraShotLatencyTest.java
new file mode 100644
index 0000000..f72ded7
--- /dev/null
+++ b/src/com/android/media/tests/CameraShotLatencyTest.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Camera shot latency test
+ *
+ * Runs Camera device performance test to measure time from taking a shot to saving a file on disk
+ * and time from shutter to shutter in fully saturated case.
+ */
+@OptionClass(alias = "camera-shot-latency")
+public class CameraShotLatencyTest extends CameraTestBase {
+
+ private static final Pattern STATS_REGEX = Pattern.compile(
+ "^(?<average>[0-9.]+)\\|(?<values>[0-9 .-]+)");
+
+ public CameraShotLatencyTest() {
+ setTestPackage("com.google.android.camera");
+ setTestClass("com.android.camera.latency.CameraShotLatencyTest");
+ setTestRunner("android.test.InstrumentationTestRunner");
+ setRuKey("CameraShotLatency");
+ setTestTimeoutMs(60 * 60 * 1000); // 1 hour
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ runInstrumentationTest(listener, new CollectingListener(listener));
+ }
+
+ /**
+ * A listener to collect the output from test run and fatal errors
+ */
+ private class CollectingListener extends DefaultCollectingListener {
+
+ public CollectingListener(ITestInvocationListener listener) {
+ super(listener);
+ }
+
+ @Override
+ public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) {
+ // Test metrics accumulated will be posted at the end of test run.
+ getAggregatedMetrics().putAll(parseResults(test.getTestName(), testMetrics));
+ }
+
+ public Map<String, String> parseResults(String testName, Map<String, String> testMetrics) {
+ // Parse shot latency stats from the instrumentation result.
+ // Format : <metric_key>=<average_of_latency>|<raw_data>
+ // Example:
+ // ElapsedTimeMs=1805|1725 1747 ... 2078
+ Map<String, String> parsed = new HashMap<String, String>();
+ for (Map.Entry<String, String> metric : testMetrics.entrySet()) {
+ Matcher matcher = STATS_REGEX.matcher(metric.getValue());
+ if (matcher.matches()) {
+ // Key name consists of a pair of test name and metric name.
+ String keyName = String.format("%s_%s", testName, metric.getKey());
+ parsed.put(keyName, matcher.group("average"));
+ } else {
+ CLog.w(String.format("Stats not in correct format: %s", metric.getValue()));
+ }
+ }
+ return parsed;
+ }
+ }
+}
diff --git a/src/com/android/media/tests/CameraShotToShotLatencyTest.java b/src/com/android/media/tests/CameraShotToShotLatencyTest.java
new file mode 100644
index 0000000..c30d5c0
--- /dev/null
+++ b/src/com/android/media/tests/CameraShotToShotLatencyTest.java
@@ -0,0 +1,184 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
+import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.BugreportCollector;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+
+import org.junit.Assert;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class CameraShotToShotLatencyTest implements IDeviceTest, IRemoteTest {
+
+ private static final Pattern MEAN_PATTERN =
+ Pattern.compile("(Shot to shot latency - mean:)(\\s*)(\\d+\\.\\d*)");
+ private static final Pattern STANDARD_DEVIATION_PATTERN =
+ Pattern.compile("(Shot to shot latency - standard deviation:)(\\s*)(\\d+\\.\\d*)");
+
+ private static final String TEST_CLASS_NAME = "com.android.camera.stress.ShotToShotLatency";
+ private static final String TEST_PACKAGE_NAME = "com.google.android.camera.tests";
+ private static final String TEST_RUNNER_NAME = "android.test.InstrumentationTestRunner";
+
+ private static final String LATENCY_KEY_MEAN = "Shot2ShotLatencyMean";
+ private static final String LATENCY_KEY_SD = "Shot2ShotLatencySD";
+ private static final String TEST_RU = "CameraLatency";
+
+ private final String mOutputPath = "mediaStressOut.txt";
+ ITestDevice mTestDevice = null;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ Assert.assertNotNull(mTestDevice);
+
+ IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME,
+ TEST_RUNNER_NAME, mTestDevice.getIDevice());
+ runner.setClassName(TEST_CLASS_NAME);
+
+ BugreportCollector bugListener = new BugreportCollector(listener, mTestDevice);
+ bugListener.addPredicate(BugreportCollector.AFTER_FAILED_TESTCASES);
+ bugListener.setDescriptiveName(this.getClass().getName());
+ Assert.assertTrue(mTestDevice.runInstrumentationTests(runner, bugListener));
+
+ Map<String, String> metrics = parseOutputFile();
+ reportMetrics(bugListener, TEST_RU, metrics);
+ cleanupDevice();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mTestDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ITestDevice getDevice() {
+ return mTestDevice;
+ }
+
+ /**
+ * Wipes the device's external memory of test collateral from prior runs.
+ * Note that all photos on the test device will be removed.
+ * @throws DeviceNotAvailableException If the device is unavailable or
+ * something happened while deleting files
+ */
+ private void cleanupDevice() throws DeviceNotAvailableException {
+ String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+ mTestDevice.executeShellCommand(String.format("rm -r %s/DCIM", extStore));
+ mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath));
+ }
+
+ /**
+ * Parses the output file generated by the underlying instrumentation test
+ * and returns the metrics to the main driver for later reporting.
+ * @return The {@link Map} that contains metrics for the test.
+ * @throws DeviceNotAvailableException If the device is unavailable or
+ * something happened while deleting files
+ */
+ private Map<String, String> parseOutputFile() throws DeviceNotAvailableException {
+ BufferedReader reader = null;
+ File outputFile = null;
+ String lineMean = null, lineSd = null;
+ Matcher m = null;
+ Map<String, String> metrics = new HashMap<String, String>();
+
+ // Read in data
+ // Output file is only 2 lines and should look something like:
+ // "Shot to shot latency - mean: 1234.5678901"
+ // "Shot to shot latency - standard deviation: 123.45678901"
+ try {
+ outputFile = mTestDevice.pullFileFromExternal(mOutputPath);
+ reader = new BufferedReader(new FileReader(outputFile));
+
+ lineMean = reader.readLine();
+ lineSd = reader.readLine();
+
+ if ((lineMean == null) || (lineSd == null)) {
+ CLog.e(String.format("Unable to find output data; hit EOF: \nmean:%s\nsd:%s",
+ lineMean, lineSd));
+ } else {
+ m = MEAN_PATTERN.matcher(lineMean);
+ if (m.matches()) {
+ metrics.put(LATENCY_KEY_MEAN, m.group(3));
+ } else {
+ CLog.e(String.format("Unable to find mean: %s", lineMean));
+ }
+
+ m = STANDARD_DEVIATION_PATTERN.matcher(lineSd);
+ if (m.matches()) {
+ metrics.put(LATENCY_KEY_SD, m.group(3));
+ } else {
+ CLog.e(String.format("Unable to find standard deviation: %s", lineSd));
+ }
+ }
+ } catch (IOException e) {
+ CLog.e(String.format("IOException reading from file: %s", e.toString()));
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ CLog.e(String.format("IOException closing file: %s", e.toString()));
+ }
+ }
+ }
+
+ return metrics;
+ }
+
+ /**
+ * Report run metrics by creating an empty test run to stick them in.
+ * @param listener The {@link ITestInvocationListener} of test results
+ * @param runName The test name
+ * @param metrics The {@link Map} that contains metrics for the given test
+ */
+ private void reportMetrics(ITestInvocationListener listener, String runName,
+ Map<String, String> metrics) {
+ InputStreamSource bugreport = mTestDevice.getBugreport();
+ listener.testLog("bugreport", LogDataType.BUGREPORT, bugreport);
+ bugreport.cancel();
+
+ CLog.d(String.format("About to report metrics: %s", metrics));
+ listener.testRunStarted(runName, 0);
+ listener.testRunEnded(0, metrics);
+ }
+}
diff --git a/src/com/android/media/tests/CameraStartupTest.java b/src/com/android/media/tests/CameraStartupTest.java
new file mode 100644
index 0000000..c19cd2f
--- /dev/null
+++ b/src/com/android/media/tests/CameraStartupTest.java
@@ -0,0 +1,236 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.OptionClass;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.targetprep.BuildError;
+import com.android.tradefed.targetprep.ITargetPreparer;
+import com.android.tradefed.targetprep.TargetSetupError;
+import com.android.tradefed.targetprep.TemperatureThrottlingWaiter;
+import com.android.tradefed.util.MultiMap;
+
+import org.junit.Assert;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Camera app startup test
+ *
+ * Runs CameraActivityTest to measure Camera startup time and reports the metrics.
+ */
+@OptionClass(alias = "camera-startup")
+public class CameraStartupTest extends CameraTestBase {
+
+ private static final Pattern STATS_REGEX = Pattern.compile(
+ "^(?<coldStartup>[0-9.]+)\\|(?<warmStartup>[0-9.]+)\\|(?<values>[0-9 .-]+)");
+ private static final String PREFIX_COLD_STARTUP = "Cold";
+ // all metrics are expected to be less than 10 mins and greater than 0.
+ private static final int METRICS_MAX_THRESHOLD_MS = 10 * 60 * 1000;
+ private static final int METRICS_MIN_THRESHOLD_MS = 0;
+ private static final String INVALID_VALUE = "-1";
+
+ @Option(name="num-test-runs", description="The number of test runs. A instrumentation "
+ + "test will be repeatedly executed. Then it posts the average of test results.")
+ private int mNumTestRuns = 1;
+
+ @Option(name="delay-between-test-runs", description="Time delay between multiple test runs, "
+ + "in msecs. Used to wait for device to cool down. "
+ + "Note that this will be ignored when TemperatureThrottlingWaiter is configured.")
+ private long mDelayBetweenTestRunsMs = 120 * 1000; // 2 minutes
+
+ private MultiMap<String, String> mMultipleRunMetrics = new MultiMap<String, String>();
+ private Map<String, String> mAverageMultipleRunMetrics = new HashMap<String, String>();
+ private long mTestRunsDurationMs = 0;
+
+ public CameraStartupTest() {
+ setTestPackage("com.google.android.camera");
+ setTestClass("com.android.camera.latency.CameraStartupTest");
+ setTestRunner("android.test.InstrumentationTestRunner");
+ setRuKey("CameraAppStartup");
+ setTestTimeoutMs(60 * 60 * 1000); // 1 hour
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ runMultipleInstrumentationTests(listener, mNumTestRuns);
+ }
+
+ private void runMultipleInstrumentationTests(ITestInvocationListener listener, int numTestRuns)
+ throws DeviceNotAvailableException {
+ Assert.assertTrue(numTestRuns > 0);
+
+ mTestRunsDurationMs = 0;
+ for (int i = 0; i < numTestRuns; ++i) {
+ CLog.v("Running multiple instrumentation tests... [%d/%d]", i + 1, numTestRuns);
+ CollectingListener singleRunListener = new CollectingListener(listener);
+ runInstrumentationTest(listener, singleRunListener);
+ mTestRunsDurationMs += getTestDurationMs();
+
+ if (singleRunListener.hasFailedTests() ||
+ singleRunListener.hasTestRunFatalError()) {
+ exitTestRunsOnError(listener, singleRunListener.getErrorMessage());
+ return;
+ }
+ if (i + 1 < numTestRuns) { // Skipping preparation on the last run
+ postSetupTestRun();
+ }
+ }
+
+ // Post the average of metrics collected in multiple instrumentation test runs.
+ postMultipleRunMetrics(listener);
+ CLog.v("multiple instrumentation tests end");
+ }
+
+ private void exitTestRunsOnError(ITestInvocationListener listener, String errorMessage) {
+ CLog.e("The instrumentation result not found. Test runs may have failed due to exceptions."
+ + " Test results will not be posted. errorMsg: %s", errorMessage);
+ listener.testRunFailed(errorMessage);
+ listener.testRunEnded(mTestRunsDurationMs, Collections.<String, String>emptyMap());
+ }
+
+ private void postMultipleRunMetrics(ITestInvocationListener listener) {
+ listener.testRunEnded(mTestRunsDurationMs, getAverageMultipleRunMetrics());
+ }
+
+ private void postSetupTestRun() throws DeviceNotAvailableException {
+ // Reboot for a cold start up of Camera application
+ CLog.d("Cold start: Rebooting...");
+ getDevice().reboot();
+
+ // Wait for device to cool down to target temperature
+ // Use TemperatureThrottlingWaiter if configured, otherwise just wait for
+ // a specific amount of time.
+ CLog.d("Cold start: Waiting for device to cool down...");
+ boolean usedTemperatureThrottlingWaiter = false;
+ for (ITargetPreparer preparer : mConfiguration.getTargetPreparers()) {
+ if (preparer instanceof TemperatureThrottlingWaiter) {
+ usedTemperatureThrottlingWaiter = true;
+ try {
+ preparer.setUp(getDevice(), null);
+ } catch (TargetSetupError e) {
+ CLog.w("No-op even when temperature is still high after wait timeout. "
+ + "error: %s", e.getMessage());
+ } catch (BuildError e) {
+ // This should not happen.
+ }
+ }
+ }
+ if (!usedTemperatureThrottlingWaiter) {
+ getRunUtil().sleep(mDelayBetweenTestRunsMs);
+ }
+ CLog.d("Device gets prepared for the next test run.");
+ }
+
+ // Call this function once at the end to get the average.
+ private Map<String, String> getAverageMultipleRunMetrics() {
+ Assert.assertTrue(mMultipleRunMetrics.size() > 0);
+
+ Set<String> keys = mMultipleRunMetrics.keySet();
+ mAverageMultipleRunMetrics.clear();
+ for (String key : keys) {
+ int sum = 0;
+ int size = 0;
+ boolean isInvalid = false;
+ for (String valueString : mMultipleRunMetrics.get(key)) {
+ int value = Integer.parseInt(valueString);
+ // If value is out of valid range, skip posting the result associated with the key
+ if (value > METRICS_MAX_THRESHOLD_MS || value < METRICS_MIN_THRESHOLD_MS) {
+ isInvalid = true;
+ break;
+ }
+ sum += value;
+ ++size;
+ }
+
+ String valueString = INVALID_VALUE;
+ if (isInvalid) {
+ CLog.w("Value is out of valid range. Key: %s ", key);
+ } else {
+ valueString = String.format("%d", (sum / size));
+ }
+ mAverageMultipleRunMetrics.put(key, valueString);
+ }
+ return mAverageMultipleRunMetrics;
+ }
+
+ /**
+ * A listener to collect the output from test run and fatal errors
+ */
+ private class CollectingListener extends DefaultCollectingListener {
+
+ public CollectingListener(ITestInvocationListener listener) {
+ super(listener);
+ }
+
+ @Override
+ public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) {
+ // Test metrics accumulated will be posted at the end of test run.
+ getAggregatedMetrics().putAll(parseResults(testMetrics));
+ }
+
+ @Override
+ public void handleTestRunEnded(ITestInvocationListener listener, long elapsedTime,
+ Map<String, String> runMetrics) {
+ // Do not post aggregated metrics from a single run to a dashboard. Instead, it needs
+ // to collect all metrics from multiple test runs.
+ mMultipleRunMetrics.putAll(getAggregatedMetrics());
+ }
+
+ public Map<String, String> parseResults(Map<String, String> testMetrics) {
+ // Parse activity time stats from the instrumentation result.
+ // Format : <metric_key>=<cold_startup>|<average_of_warm_startups>|<all_startups>
+ // Example:
+ // VideoStartupTimeMs=1098|1184.6|1098 1222 ... 788
+ // VideoOnCreateTimeMs=138|103.3|138 114 ... 114
+ // VideoOnResumeTimeMs=39|40.4|39 36 ... 41
+ // VideoFirstPreviewFrameTimeMs=0|0.0|0 0 ... 0
+ // CameraStartupTimeMs=2388|1045.4|2388 1109 ... 746
+ // CameraOnCreateTimeMs=574|122.7|574 124 ... 109
+ // CameraOnResumeTimeMs=610|504.6|610 543 ... 278
+ // CameraFirstPreviewFrameTimeMs=0|0.0|0 0 ... 0
+ //
+ // Then report only the first two startup time of cold startup and average warm startup.
+ Map<String, String> parsed = new HashMap<String, String>();
+ for (Map.Entry<String, String> metric : testMetrics.entrySet()) {
+ Matcher matcher = STATS_REGEX.matcher(metric.getValue());
+ String keyName = metric.getKey();
+ String coldStartupValue = INVALID_VALUE;
+ String warmStartupValue = INVALID_VALUE;
+ if (matcher.matches()) {
+ coldStartupValue = matcher.group("coldStartup");
+ warmStartupValue = matcher.group("warmStartup");
+ }
+ parsed.put(PREFIX_COLD_STARTUP + keyName, coldStartupValue);
+ parsed.put(keyName, warmStartupValue);
+ }
+ return parsed;
+ }
+ }
+}
diff --git a/src/com/android/media/tests/CameraStressTest.java b/src/com/android/media/tests/CameraStressTest.java
new file mode 100644
index 0000000..def3f39
--- /dev/null
+++ b/src/com/android/media/tests/CameraStressTest.java
@@ -0,0 +1,513 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.Log;
+import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
+import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.result.CollectingTestListener;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.RegexTrie;
+import com.android.tradefed.util.StreamUtil;
+
+import junit.framework.TestCase;
+
+import org.junit.Assert;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Runs the Camera stress testcases.
+ * FIXME: more details
+ * <p/>
+ * Note that this test will not run properly unless /sdcard is mounted and writable.
+ */
+public class CameraStressTest implements IDeviceTest, IRemoteTest {
+ private static final String LOG_TAG = "CameraStressTest";
+
+ ITestDevice mTestDevice = null;
+
+ // Constants for running the tests
+ private static final String TEST_PACKAGE_NAME = "com.google.android.camera.tests";
+ private static final String TEST_RUNNER = "com.android.camera.stress.CameraStressTestRunner";
+
+ //Max test timeout - 3 hrs
+ private static final int MAX_TEST_TIMEOUT = 3 * 60 * 60 * 1000;
+
+ private final String mOutputPath = "mediaStressOut.txt";
+
+ /**
+ * Stores the test cases that we should consider running.
+ *
+ * <p>This currently consists of "startup" and "latency"
+ */
+ private List<TestInfo> mTestCases = new ArrayList<>();
+
+ // Options for the running the gCam test
+ @Option(name = "gCam", description = "Run gCam back image capture test")
+ private boolean mGcam = false;
+
+ /**
+ * A struct that contains useful info about the tests to run
+ */
+ static class TestInfo {
+ public String mTestName = null;
+ public String mClassName = null;
+ public String mTestMetricsName = null;
+ public Map<String, String> mInstrumentationArgs = new HashMap<>();
+ public RegexTrie<String> mPatternMap = new RegexTrie<>();
+
+ @Override
+ public String toString() {
+ return String.format("TestInfo: name(%s) class(%s) metric(%s) patterns(%s)", mTestName,
+ mClassName, mTestMetricsName, mPatternMap);
+ }
+ }
+
+ /**
+ * Set up the pattern map for parsing output files
+ * <p/>
+ * Exposed for unit meta-testing
+ */
+ static RegexTrie<String> getPatternMap() {
+ RegexTrie<String> patMap = new RegexTrie<>();
+ patMap.put("SwitchPreview", "^Camera Switch Mode:");
+
+ // For versions of the on-device test that don't differentiate between front and back camera
+ patMap.put("ImageCapture", "^Camera Image Capture");
+ patMap.put("VideoRecording", "^Camera Video Capture");
+
+ // For versions that do differentiate
+ patMap.put("FrontImageCapture", "^Front Camera Image Capture");
+ patMap.put("ImageCapture", "^Back Camera Image Capture");
+ patMap.put("FrontVideoRecording", "^Front Camera Video Capture");
+ patMap.put("VideoRecording", "^Back Camera Video Capture");
+
+ // Actual metrics to collect for a given key
+ patMap.put("loopCount", "^No of loops :(\\d+)");
+ patMap.put("iters", "^loop:.+,(\\d+)");
+
+ return patMap;
+ }
+
+ /**
+ * Set up the configurations for the test cases we want to run
+ */
+ private void testInfoSetup() {
+ RegexTrie<String> patMap = getPatternMap();
+ TestInfo t = new TestInfo();
+
+ if (mGcam) {
+ // Back Image capture stress test for gCam
+ t.mTestName = "testBackImageCapture";
+ t.mClassName = "com.android.camera.stress.ImageCapture";
+ t.mTestMetricsName = "GCamApplicationStress";
+ t.mInstrumentationArgs.put("image_iterations", Integer.toString(100));
+ t.mPatternMap = patMap;
+ mTestCases.add(t);
+
+ } else {
+ // Image capture stress test
+ t.mTestName = "imagecap";
+ t.mClassName = "com.android.camera.stress.ImageCapture";
+ t.mTestMetricsName = "CameraApplicationStress";
+ t.mInstrumentationArgs.put("image_iterations", Integer.toString(100));
+ t.mPatternMap = patMap;
+ mTestCases.add(t);
+
+ // Image capture stress test
+ t = new TestInfo();
+ t.mTestName = "videocap";
+ t.mClassName = "com.android.camera.stress.VideoCapture";
+ t.mTestMetricsName = "CameraApplicationStress";
+ t.mInstrumentationArgs.put("video_iterations", Integer.toString(100));
+ t.mPatternMap = patMap;
+ mTestCases.add(t);
+
+ // "SwitchPreview" stress test
+ t = new TestInfo();
+ t.mTestName = "switch";
+ t.mClassName = "com.android.camera.stress.SwitchPreview";
+ t.mTestMetricsName = "CameraApplicationStress";
+ t.mPatternMap = patMap;
+ mTestCases.add(t);
+ }
+ }
+
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ Assert.assertNotNull(mTestDevice);
+ testInfoSetup();
+ for (TestInfo test : mTestCases) {
+ cleanTmpFiles();
+ executeTest(test, listener);
+ logOutputFiles(test, listener);
+ }
+
+ cleanTmpFiles();
+ }
+
+ private void executeTest(TestInfo test, ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME,
+ TEST_RUNNER, mTestDevice.getIDevice());
+ CollectingTestListener auxListener = new CollectingTestListener();
+
+ runner.setClassName(test.mClassName);
+ runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS);
+ if (mGcam){
+ runner.setMethodName(test.mClassName, test.mTestName);
+ }
+
+ Set<String> argumentKeys = test.mInstrumentationArgs.keySet();
+ for (String s : argumentKeys) {
+ runner.addInstrumentationArg(s, test.mInstrumentationArgs.get(s));
+ }
+
+ mTestDevice.runInstrumentationTests(runner, listener, auxListener);
+
+ // Grab a bugreport if warranted
+ if (auxListener.hasFailedTests()) {
+ Log.e(LOG_TAG, String.format("Grabbing bugreport after test '%s' finished with " +
+ "%d failures.", test.mTestName, auxListener.getNumAllFailedTests()));
+ InputStreamSource bugreport = mTestDevice.getBugreport();
+ listener.testLog(String.format("bugreport-%s.txt", test.mTestName),
+ LogDataType.BUGREPORT, bugreport);
+ bugreport.cancel();
+ }
+ }
+
+ /**
+ * Clean up temp files from test runs
+ * <p />
+ * Note that all photos on the test device will be removed
+ */
+ private void cleanTmpFiles() throws DeviceNotAvailableException {
+ String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+ mTestDevice.executeShellCommand(String.format("rm -r %s/DCIM/Camera", extStore));
+ mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath));
+ }
+
+ /**
+ * Pull the output file from the device, add it to the logs, and also parse out the relevant
+ * test metrics and report them. Additionally, pull the memory file (if it exists) and report
+ * it.
+ */
+ private void logOutputFiles(TestInfo test, ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ File outputFile = null;
+ InputStreamSource outputSource = null;
+ try {
+ outputFile = mTestDevice.pullFileFromExternal(mOutputPath);
+
+ if (outputFile == null) {
+ return;
+ }
+
+ // Upload a verbatim copy of the output file
+ Log.d(LOG_TAG, String.format("Sending %d byte file %s into the logosphere!",
+ outputFile.length(), outputFile));
+ outputSource = new FileInputStreamSource(outputFile);
+ listener.testLog(String.format("output-%s.txt", test.mTestName), LogDataType.TEXT,
+ outputSource);
+
+ // Parse the output file to upload aggregated metrics
+ parseOutputFile(test, new FileInputStream(outputFile), listener);
+ } catch (IOException e) {
+ Log.e(LOG_TAG, String.format("IOException while reading or parsing output file: %s", e));
+ } finally {
+ FileUtil.deleteFile(outputFile);
+ StreamUtil.cancel(outputSource);
+ }
+ }
+
+ /**
+ * Parse the relevant metrics from the Instrumentation test output file
+ */
+ private void parseOutputFile(TestInfo test, InputStream dataStream,
+ ITestInvocationListener listener) {
+ Map<String, String> runMetrics = new HashMap<>();
+
+ String contents;
+ try {
+ contents = StreamUtil.getStringFromStream(dataStream);
+ } catch (IOException e) {
+ Log.e(LOG_TAG, String.format("Got IOException during %s test processing: %s",
+ test.mTestName, e));
+ return;
+ }
+
+ String key = null;
+ Integer countExpected = null;
+ Integer countActual = null;
+
+ List<String> lines = Arrays.asList(contents.split("\n"));
+ ListIterator<String> lineIter = lines.listIterator();
+ String line;
+ while (lineIter.hasNext()) {
+ line = lineIter.next();
+ List<List<String>> capture = new ArrayList<>(1);
+ String pattern = test.mPatternMap.retrieve(capture, line);
+ if (pattern != null) {
+ if ("loopCount".equals(pattern)) {
+ // First capture in first (only) string
+ countExpected = Integer.parseInt(capture.get(0).get(0));
+ } else if ("iters".equals(pattern)) {
+ // First capture in first (only) string
+ countActual = Integer.parseInt(capture.get(0).get(0));
+
+ if (countActual != null) {
+ // countActual starts counting at 0
+ countActual += 1;
+ }
+ } else {
+ // Assume that the pattern is the name of a key
+
+ // commit, if there was a previous key
+ if (key != null) {
+ int value = coalesceLoopCounts(countActual, countExpected);
+ runMetrics.put(key, Integer.toString(value));
+ }
+
+ key = pattern;
+ countExpected = null;
+ countActual = null;
+ }
+
+ Log.d(LOG_TAG, String.format("Got %s key '%s' and captures '%s'",
+ test.mTestName, key, capture.toString()));
+ } else if (line.isEmpty()) {
+ // ignore
+ continue;
+ } else {
+ Log.e(LOG_TAG, String.format("Got unmatched line: %s", line));
+ continue;
+ }
+
+ // commit the final key, if there was one
+ if (key != null) {
+ int value = coalesceLoopCounts(countActual, countExpected);
+ runMetrics.put(key, Integer.toString(value));
+ }
+ }
+
+ reportMetrics(listener, test, runMetrics);
+ }
+
+ /**
+ * Given an actual and an expected iteration count, determine a single metric to report.
+ */
+ private int coalesceLoopCounts(Integer actual, Integer expected) {
+ if (expected == null || expected <= 0) {
+ return -1;
+ } else if (actual == null) {
+ return expected;
+ } else {
+ return actual;
+ }
+ }
+
+ /**
+ * Report run metrics by creating an empty test run to stick them in
+ * <p />
+ * Exposed for unit testing
+ */
+ void reportMetrics(ITestInvocationListener listener, TestInfo test,
+ Map<String, String> metrics) {
+ // Create an empty testRun to report the parsed runMetrics
+ Log.e(LOG_TAG, String.format("About to report metrics for %s: %s", test.mTestMetricsName,
+ metrics));
+ listener.testRunStarted(test.mTestMetricsName, 0);
+ listener.testRunEnded(0, metrics);
+ }
+
+ @Override
+ public void setDevice(ITestDevice device) {
+ mTestDevice = device;
+ }
+
+ @Override
+ public ITestDevice getDevice() {
+ return mTestDevice;
+ }
+
+ /**
+ * A meta-test to ensure that bits of the BluetoothStressTest are working properly
+ */
+ public static class MetaTest extends TestCase {
+ private CameraStressTest mTestInstance = null;
+
+ private TestInfo mTestInfo = null;
+
+ private TestInfo mReportedTestInfo = null;
+ private Map<String, String> mReportedMetrics = null;
+
+ private static String join(String... pieces) {
+ StringBuilder sb = new StringBuilder();
+ for (String piece : pieces) {
+ sb.append(piece);
+ sb.append("\n");
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void setUp() throws Exception {
+ mTestInstance = new CameraStressTest() {
+ @Override
+ void reportMetrics(ITestInvocationListener l, TestInfo test,
+ Map<String, String> metrics) {
+ mReportedTestInfo = test;
+ mReportedMetrics = metrics;
+ }
+ };
+
+ // Image capture stress test
+ mTestInfo = new TestInfo();
+ TestInfo t = mTestInfo; // for convenience
+ t.mTestName = "capture";
+ t.mClassName = "com.android.camera.stress.ImageCapture";
+ t.mTestMetricsName = "camera_application_stress";
+ t.mPatternMap = getPatternMap();
+ }
+
+ /**
+ * Make sure that parsing works for devices sending output in the old format
+ */
+ public void testParse_old() throws Exception {
+ String output = join(
+ "Camera Image Capture",
+ "No of loops :100",
+ "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " +
+ ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 " +
+ ",36 ,37 ,38 ,39 ,40 ,41 ,42",
+ "Camera Video Capture",
+ "No of loops :100",
+ "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " +
+ ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 " +
+ ",36 ,37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 " +
+ ",53 ,54 ,55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 " +
+ ",70 ,71 ,72 ,73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 " +
+ ",87 ,88 ,89 ,90 ,91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99",
+ "Camera Switch Mode:",
+ "No of loops :200",
+ "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13");
+
+ InputStream iStream = new ByteArrayInputStream(output.getBytes());
+ mTestInstance.parseOutputFile(mTestInfo, iStream, null);
+ assertEquals(mTestInfo, mReportedTestInfo);
+ assertNotNull(mReportedMetrics);
+ Log.e(LOG_TAG, String.format("Got reported metrics: %s", mReportedMetrics.toString()));
+ assertEquals(3, mReportedMetrics.size());
+ assertEquals("43", mReportedMetrics.get("ImageCapture"));
+ assertEquals("100", mReportedMetrics.get("VideoRecording"));
+ assertEquals("14", mReportedMetrics.get("SwitchPreview"));
+ }
+
+ /**
+ * Make sure that parsing works for devices sending output in the new format
+ */
+ public void testParse_new() throws Exception {
+ String output = join(
+ "Camera Stress Test result",
+ "/folder/subfolder/data/CameraStressTest_git_honeycomb-mr1-release_" +
+ "1700614441c02617_109535_CameraStressOut.txt",
+ "Back Camera Image Capture",
+ "No of loops :100",
+ "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " +
+ ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " +
+ ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " +
+ ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " +
+ ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " +
+ ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99",
+ "Front Camera Image Capture",
+ "No of loops :100",
+ "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " +
+ ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " +
+ ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " +
+ ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " +
+ ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " +
+ ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98",
+ "Back Camera Video Capture",
+ "No of loops :100",
+ "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " +
+ ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " +
+ ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " +
+ ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " +
+ ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " +
+ ",91 ,92 ,93 ,94 ,95 ,96 ,97",
+ "Front Camera Video Capture",
+ "No of loops :100",
+ "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " +
+ ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " +
+ ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " +
+ ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " +
+ ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " +
+ ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99",
+ "Camera Switch Mode:",
+ "No of loops :200",
+ "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " +
+ ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " +
+ ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " +
+ ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " +
+ ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " +
+ ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99 ,100 ,101 ,102 ,103 ,104 ,105 ,106 " +
+ ",107 ,108 ,109 ,110 ,111 ,112 ,113 ,114 ,115 ,116 ,117 ,118 ,119 ,120 " +
+ ",121 ,122 ,123 ,124 ,125 ,126 ,127 ,128 ,129 ,130 ,131 ,132 ,133 ,134 " +
+ ",135 ,136 ,137 ,138 ,139 ,140 ,141 ,142 ,143 ,144 ,145 ,146 ,147 ,148 " +
+ ",149 ,150 ,151 ,152 ,153 ,154 ,155 ,156 ,157 ,158 ,159 ,160 ,161 ,162 " +
+ ",163 ,164 ,165 ,166 ,167 ,168 ,169 ,170 ,171 ,172 ,173 ,174 ,175 ,176 " +
+ ",177 ,178 ,179 ,180 ,181 ,182 ,183 ,184 ,185 ,186 ,187 ,188 ,189 ,190 " +
+ ",191 ,192 ,193 ,194 ,195 ,196 ,197 ,198 ,199");
+
+ InputStream iStream = new ByteArrayInputStream(output.getBytes());
+ mTestInstance.parseOutputFile(mTestInfo, iStream, null);
+ assertEquals(mTestInfo, mReportedTestInfo);
+ assertNotNull(mReportedMetrics);
+ Log.e(LOG_TAG, String.format("Got reported metrics: %s", mReportedMetrics.toString()));
+ assertEquals(5, mReportedMetrics.size());
+ assertEquals("100", mReportedMetrics.get("ImageCapture"));
+ assertEquals("99", mReportedMetrics.get("FrontImageCapture"));
+ assertEquals("98", mReportedMetrics.get("VideoRecording"));
+ assertEquals("100", mReportedMetrics.get("FrontVideoRecording"));
+ assertEquals("200", mReportedMetrics.get("SwitchPreview"));
+ }
+ }
+}
+
diff --git a/src/com/android/media/tests/CameraTestBase.java b/src/com/android/media/tests/CameraTestBase.java
new file mode 100644
index 0000000..8714a22
--- /dev/null
+++ b/src/com/android/media/tests/CameraTestBase.java
@@ -0,0 +1,840 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.CollectingOutputReceiver;
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.IConfiguration;
+import com.android.tradefed.config.IConfigurationReceiver;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ByteArrayInputStreamSource;
+import com.android.tradefed.result.CollectingTestListener;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.testtype.InstrumentationTest;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.RunUtil;
+import com.android.tradefed.util.StreamUtil;
+
+import org.junit.Assert;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Camera test base class
+ *
+ * Camera2StressTest, CameraStartupTest, Camera2LatencyTest and CameraPerformanceTest use this base
+ * class for Camera ivvavik and later.
+ */
+public class CameraTestBase implements IDeviceTest, IRemoteTest, IConfigurationReceiver {
+
+ private static final long SHELL_TIMEOUT_MS = 60 * 1000; // 1 min
+ private static final int SHELL_MAX_ATTEMPTS = 3;
+ protected static final String PROCESS_CAMERA_DAEMON = "mm-qcamera-daemon";
+ protected static final String PROCESS_MEDIASERVER = "mediaserver";
+ protected static final String PROCESS_CAMERA_APP = "com.google.android.GoogleCamera";
+ protected static final String DUMP_ION_HEAPS_COMMAND = "cat /d/ion/heaps/system";
+ protected static final String ARGUMENT_TEST_ITERATIONS = "iterations";
+
+ @Option(name = "test-package", description = "Test package to run.")
+ private String mTestPackage = "com.google.android.camera";
+
+ @Option(name = "test-class", description = "Test class to run.")
+ private String mTestClass = null;
+
+ @Option(name = "test-methods", description = "Test method to run. May be repeated.")
+ private Collection<String> mTestMethods = new ArrayList<>();
+
+ @Option(name = "test-runner", description = "Test runner for test instrumentation.")
+ private String mTestRunner = "android.test.InstrumentationTestRunner";
+
+ @Option(name = "test-timeout", description = "Max time allowed in ms for a test run.")
+ private int mTestTimeoutMs = 60 * 60 * 1000; // 1 hour
+
+ @Option(name = "shell-timeout",
+ description="The defined timeout (in milliseconds) is used as a maximum waiting time "
+ + "when expecting the command output from the device. At any time, if the "
+ + "shell command does not output anything for a period longer than defined "
+ + "timeout the TF run terminates. For no timeout, set to 0.")
+ private long mShellTimeoutMs = 60 * 60 * 1000; // default to 1 hour
+
+ @Option(name = "ru-key", description = "Result key to use when posting to the dashboard.")
+ private String mRuKey = null;
+
+ @Option(name = "logcat-on-failure", description =
+ "take a logcat snapshot on every test failure.")
+ private boolean mLogcatOnFailure = false;
+
+ @Option(
+ name = "instrumentation-arg",
+ description = "Additional instrumentation arguments to provide."
+ )
+ private Map<String, String> mInstrArgMap = new HashMap<>();
+
+ @Option(name = "dump-meminfo", description =
+ "take a dumpsys meminfo at a given interval time.")
+ private boolean mDumpMeminfo = false;
+
+ @Option(name="dump-meminfo-interval-ms",
+ description="Interval of calling dumpsys meminfo in milliseconds.")
+ private int mMeminfoIntervalMs = 5 * 60 * 1000; // 5 minutes
+
+ @Option(name = "dump-ion-heap", description =
+ "dump ION allocations at the end of test.")
+ private boolean mDumpIonHeap = false;
+
+ @Option(name = "dump-thread-count", description =
+ "Count the number of threads in Camera process at a given interval time.")
+ private boolean mDumpThreadCount = false;
+
+ @Option(name="dump-thread-count-interval-ms",
+ description="Interval of calling ps to count the number of threads in milliseconds.")
+ private int mThreadCountIntervalMs = 5 * 60 * 1000; // 5 minutes
+
+ @Option(name="iterations", description="The number of iterations to run. Default to 1. "
+ + "This takes effect only when Camera2InstrumentationTestRunner is used to execute "
+ + "framework stress tests.")
+ private int mIterations = 1;
+
+ private ITestDevice mDevice = null;
+
+ // A base listener to collect the results from each test run. Test results will be forwarded
+ // to other listeners.
+ private AbstractCollectingListener mCollectingListener = null;
+
+ private long mStartTimeMs = 0;
+
+ private MeminfoTimer mMeminfoTimer = null;
+ private ThreadTrackerTimer mThreadTrackerTimer = null;
+
+ protected IConfiguration mConfiguration;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ // ignore
+ }
+
+ /**
+ * Run Camera instrumentation test with a default listener.
+ *
+ * @param listener the ITestInvocationListener of test results
+ * @throws DeviceNotAvailableException
+ */
+ protected void runInstrumentationTest(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ if (mCollectingListener == null) {
+ mCollectingListener = new DefaultCollectingListener(listener);
+ }
+ runInstrumentationTest(listener, mCollectingListener);
+ }
+
+ /**
+ * Run Camera instrumentation test with a listener to gather the metrics from the individual
+ * test runs.
+ *
+ * @param listener the ITestInvocationListener of test results
+ * @param collectingListener the {@link CollectingTestListener} to collect the metrics from
+ * test runs
+ * @throws DeviceNotAvailableException
+ */
+ protected void runInstrumentationTest(ITestInvocationListener listener,
+ AbstractCollectingListener collectingListener)
+ throws DeviceNotAvailableException {
+ Assert.assertNotNull(collectingListener);
+ mCollectingListener = collectingListener;
+
+ InstrumentationTest instr = new InstrumentationTest();
+ instr.setDevice(getDevice());
+ instr.setPackageName(getTestPackage());
+ instr.setRunnerName(getTestRunner());
+ instr.setClassName(getTestClass());
+ instr.setTestTimeout(getTestTimeoutMs());
+ instr.setShellTimeout(getShellTimeoutMs());
+ instr.setLogcatOnFailure(mLogcatOnFailure);
+ instr.setRunName(getRuKey());
+ instr.setRerunMode(false);
+
+ // Set test iteration.
+ if (getIterationCount() > 1) {
+ CLog.v("Setting test iterations: %d", getIterationCount());
+ Map<String, String> instrArgMap = getInstrumentationArgMap();
+ instrArgMap.put(ARGUMENT_TEST_ITERATIONS, String.valueOf(getIterationCount()));
+ }
+
+ for (Map.Entry<String, String> entry : getInstrumentationArgMap().entrySet()) {
+ instr.addInstrumentationArg(entry.getKey(), entry.getValue());
+ }
+
+ // Check if dumpheap needs to be taken for native processes before test runs.
+ if (shouldDumpMeminfo()) {
+ mMeminfoTimer = new MeminfoTimer(0, mMeminfoIntervalMs);
+ }
+ if (shouldDumpThreadCount()) {
+ long delayMs = mThreadCountIntervalMs / 2; // Not to run all dump at the same interval.
+ mThreadTrackerTimer = new ThreadTrackerTimer(delayMs, mThreadCountIntervalMs);
+ }
+
+ // Run tests.
+ mStartTimeMs = System.currentTimeMillis();
+ if (mTestMethods.size() > 0) {
+ CLog.d(String.format("The number of test methods is: %d", mTestMethods.size()));
+ for (String testName : mTestMethods) {
+ instr.setMethodName(testName);
+ instr.run(mCollectingListener);
+ }
+ } else {
+ instr.run(mCollectingListener);
+ }
+
+ dumpIonHeaps(mCollectingListener, getTestClass());
+ }
+
+ /**
+ * A base listener to collect all test results and metrics from Camera instrumentation test run.
+ * Abstract methods can be overridden to handle test metrics or inform of test run ended.
+ */
+ protected abstract class AbstractCollectingListener extends CollectingTestListener {
+
+ private ITestInvocationListener mListener = null;
+ private Map<String, String> mMetrics = new HashMap<>();
+ private Map<String, String> mFatalErrors = new HashMap<>();
+
+ private static final String INCOMPLETE_TEST_ERR_MSG_PREFIX =
+ "Test failed to run to completion. Reason: 'Instrumentation run failed";
+
+ public AbstractCollectingListener(ITestInvocationListener listener) {
+ mListener = listener;
+ }
+
+ /**
+ * Override only when subclasses need to get the test metrics from an individual
+ * instrumentation test. To aggregate the metrics from each test, update the
+ * getAggregatedMetrics and post them at the end of test run.
+ *
+ * @param test identifies the test
+ * @param testMetrics a {@link Map} of the metrics emitted
+ */
+ abstract public void handleMetricsOnTestEnded(TestIdentifier test,
+ Map<String, String> testMetrics);
+
+ /**
+ * Override only when it needs to inform subclasses of instrumentation test run ended,
+ * so that subclasses have a chance to peek the aggregated results at the end of test run
+ * and to decide what metrics to be posted.
+ * Either {@link ITestInvocationListener#testRunEnded} or
+ * {@link ITestInvocationListener#testRunFailed} should be called in this function to
+ * report the test run status.
+ *
+ * @param listener - the ITestInvocationListener of test results
+ * @param elapsedTime - device reported elapsed time, in milliseconds
+ * @param runMetrics - key-value pairs reported at the end of an instrumentation test run.
+ * Use getAggregatedMetrics to retrieve the metrics aggregated
+ * from an individual test, instead.
+ */
+ abstract public void handleTestRunEnded(ITestInvocationListener listener,
+ long elapsedTime, Map<String, String> runMetrics);
+
+ /**
+ * Report the end of an individual camera test and delegate handling the collected metrics
+ * to subclasses. Do not override testEnded to manipulate the test metrics after each test.
+ * Instead, use handleMetricsOnTestEnded.
+ *
+ * @param test identifies the test
+ * @param testMetrics a {@link Map} of the metrics emitted
+ */
+ @Override
+ public void testEnded(TestIdentifier test, long endTime, Map<String, String> testMetrics) {
+ super.testEnded(test, endTime, testMetrics);
+ handleMetricsOnTestEnded(test, testMetrics);
+ stopDumping(test);
+ mListener.testEnded(test, endTime, testMetrics);
+ }
+
+ @Override
+ public void testStarted(TestIdentifier test, long startTime) {
+ super.testStarted(test, startTime);
+ startDumping(test);
+ mListener.testStarted(test, startTime);
+ }
+
+ @Override
+ public void testFailed(TestIdentifier test, String trace) {
+ super.testFailed(test, trace);
+ // If the test failed to run to complete, this is an exceptional case.
+ // Let this test run fail so that it can rerun.
+ if (trace.startsWith(INCOMPLETE_TEST_ERR_MSG_PREFIX)) {
+ mFatalErrors.put(test.getTestName(), trace);
+ CLog.d("Test (%s) failed due to fatal error : %s", test.getTestName(), trace);
+ }
+ mListener.testFailed(test, trace);
+ }
+
+ @Override
+ public void testRunFailed(String errorMessage) {
+ super.testRunFailed(errorMessage);
+ mFatalErrors.put(getRuKey(), errorMessage);
+ }
+
+ @Override
+ public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) {
+ super.testRunEnded(elapsedTime, runMetrics);
+ handleTestRunEnded(mListener, elapsedTime, runMetrics);
+ // never be called since handleTestRunEnded will handle it if needed.
+ //mListener.testRunEnded(elapsedTime, runMetrics);
+ }
+
+ @Override
+ public void testRunStarted(String runName, int testCount) {
+ super.testRunStarted(runName, testCount);
+ mListener.testRunStarted(runName, testCount);
+ }
+
+ @Override
+ public void testRunStopped(long elapsedTime) {
+ super.testRunStopped(elapsedTime);
+ mListener.testRunStopped(elapsedTime);
+ }
+
+ @Override
+ public void testLog(String dataName, LogDataType dataType, InputStreamSource dataStream) {
+ super.testLog(dataName, dataType, dataStream);
+ mListener.testLog(dataName, dataType, dataStream);
+ }
+
+ protected void startDumping(TestIdentifier test) {
+ if (shouldDumpMeminfo()) {
+ mMeminfoTimer.start(test);
+ }
+ if (shouldDumpThreadCount()) {
+ mThreadTrackerTimer.start(test);
+ }
+ }
+
+ protected void stopDumping(TestIdentifier test) {
+ InputStreamSource outputSource = null;
+ File outputFile = null;
+ if (shouldDumpMeminfo()) {
+ mMeminfoTimer.stop();
+ // Grab a snapshot of meminfo file and post it to dashboard.
+ try {
+ outputFile = mMeminfoTimer.getOutputFile();
+ outputSource = new FileInputStreamSource(outputFile, true /* delete */);
+ String logName = String.format("meminfo_%s", test.getTestName());
+ mListener.testLog(logName, LogDataType.TEXT, outputSource);
+ } finally {
+ StreamUtil.cancel(outputSource);
+ }
+ }
+ if (shouldDumpThreadCount()) {
+ mThreadTrackerTimer.stop();
+ try {
+ outputFile = mThreadTrackerTimer.getOutputFile();
+ outputSource = new FileInputStreamSource(outputFile, true /* delete */);
+ String logName = String.format("ps_%s", test.getTestName());
+ mListener.testLog(logName, LogDataType.TEXT, outputSource);
+ } finally {
+ StreamUtil.cancel(outputSource);
+ }
+ }
+ }
+
+ public Map<String, String> getAggregatedMetrics() {
+ return mMetrics;
+ }
+
+ public ITestInvocationListener getListeners() {
+ return mListener;
+ }
+
+ /**
+ * Determine that the test run failed with fatal errors.
+ *
+ * @return True if test run has a failure due to fatal error.
+ */
+ public boolean hasTestRunFatalError() {
+ return (getNumTotalTests() > 0 && mFatalErrors.size() > 0);
+ }
+
+ public Map<String, String> getFatalErrors() {
+ return mFatalErrors;
+ }
+
+ public String getErrorMessage() {
+ StringBuilder sb = new StringBuilder();
+ for (Map.Entry<String, String> error : mFatalErrors.entrySet()) {
+ sb.append(error.getKey());
+ sb.append(" : ");
+ sb.append(error.getValue());
+ sb.append("\n");
+ }
+ return sb.toString();
+ }
+ }
+
+ protected class DefaultCollectingListener extends AbstractCollectingListener {
+
+ public DefaultCollectingListener(ITestInvocationListener listener) {
+ super(listener);
+ }
+
+ @Override
+ public void handleMetricsOnTestEnded(TestIdentifier test, Map<String, String> testMetrics) {
+ if (testMetrics == null) {
+ return; // No-op if there is nothing to post.
+ }
+ getAggregatedMetrics().putAll(testMetrics);
+ }
+
+ @Override
+ public void handleTestRunEnded(ITestInvocationListener listener, long elapsedTime,
+ Map<String, String> runMetrics) {
+ // Post aggregated metrics at the end of test run.
+ listener.testRunEnded(getTestDurationMs(), getAggregatedMetrics());
+ }
+ }
+
+ // TODO: Leverage AUPT to collect system logs (meminfo, ION allocations and processes/threads)
+ private class MeminfoTimer {
+
+ private static final String LOG_HEADER =
+ "uptime,pssCameraDaemon,pssCameraApp,ramTotal,ramFree,ramUsed";
+ private static final String DUMPSYS_MEMINFO_COMMAND =
+ "dumpsys meminfo -c | grep -w -e " + "^ram -e ^time";
+ private String[] mDumpsysMemInfoProc = {
+ PROCESS_CAMERA_DAEMON, PROCESS_CAMERA_APP, PROCESS_MEDIASERVER
+ };
+ private static final int STATE_STOPPED = 0;
+ private static final int STATE_SCHEDULED = 1;
+ private static final int STATE_RUNNING = 2;
+
+ private int mState = STATE_STOPPED;
+ private Timer mTimer = new Timer(true); // run as a daemon thread
+ private long mDelayMs = 0;
+ private long mPeriodMs = 60 * 1000; // 60 sec
+ private File mOutputFile = null;
+ private String mCommand;
+
+ public MeminfoTimer(long delayMs, long periodMs) {
+ mDelayMs = delayMs;
+ mPeriodMs = periodMs;
+ mCommand = DUMPSYS_MEMINFO_COMMAND;
+ for (String process : mDumpsysMemInfoProc) {
+ mCommand += " -e " + process;
+ }
+ }
+
+ synchronized void start(TestIdentifier test) {
+ if (isRunning()) {
+ stop();
+ }
+ // Create an output file.
+ if (createOutputFile(test) == null) {
+ CLog.w("Stop collecting meminfo since meminfo log file not found.");
+ mState = STATE_STOPPED;
+ return; // No-op
+ }
+ mTimer.scheduleAtFixedRate(new TimerTask() {
+ @Override
+ public void run() {
+ mState = STATE_RUNNING;
+ dumpMeminfo(mCommand, mOutputFile);
+ }
+ }, mDelayMs, mPeriodMs);
+ mState = STATE_SCHEDULED;
+ }
+
+ synchronized void stop() {
+ mState = STATE_STOPPED;
+ mTimer.cancel();
+ }
+
+ synchronized boolean isRunning() {
+ return (mState == STATE_RUNNING);
+ }
+
+ public File getOutputFile() {
+ return mOutputFile;
+ }
+
+ private File createOutputFile(TestIdentifier test) {
+ try {
+ mOutputFile = FileUtil.createTempFile(
+ String.format("meminfo_%s", test.getTestName()), "csv");
+ BufferedWriter writer = new BufferedWriter(new FileWriter(mOutputFile, false));
+ writer.write(LOG_HEADER);
+ writer.newLine();
+ writer.flush();
+ writer.close();
+ } catch (IOException e) {
+ CLog.w("Failed to create meminfo log file %s:", mOutputFile.getAbsolutePath());
+ CLog.e(e);
+ return null;
+ }
+ return mOutputFile;
+ }
+ }
+
+ void dumpMeminfo(String command, File outputFile) {
+ try {
+ CollectingOutputReceiver receiver = new CollectingOutputReceiver();
+ // Dump meminfo in a compact form.
+ getDevice().executeShellCommand(command, receiver,
+ SHELL_TIMEOUT_MS, TimeUnit.MILLISECONDS, SHELL_MAX_ATTEMPTS);
+ printMeminfo(outputFile, receiver.getOutput());
+ } catch (DeviceNotAvailableException e) {
+ CLog.w("Failed to dump meminfo:");
+ CLog.e(e);
+ }
+ }
+
+ void printMeminfo(File outputFile, String meminfo) {
+ // Parse meminfo and print each iteration in a line in a .csv format. The meminfo output
+ // are separated into three different formats:
+ //
+ // Format: time,<uptime>,<realtime>
+ // eg. "time,59459911,63354673"
+ //
+ // Format: proc,<oom_label>,<process_name>,<pid>,<pss>,<hasActivities>
+ // eg. "proc,native,mm-qcamera-daemon,522,12881,e"
+ // "proc,fore,com.google.android.GoogleCamera,26560,70880,a"
+ //
+ // Format: ram,<total>,<free>,<used>
+ // eg. "ram,1857364,810189,541516"
+ BufferedWriter writer = null;
+ BufferedReader reader = null;
+ try {
+ final String DELIMITER = ",";
+ writer = new BufferedWriter(new FileWriter(outputFile, true));
+ reader = new BufferedReader(new StringReader(meminfo));
+ String line;
+ String uptime = null;
+ String pssCameraNative = null;
+ String pssCameraApp = null;
+ String ramTotal = null;
+ String ramFree = null;
+ String ramUsed = null;
+ while ((line = reader.readLine()) != null) {
+ if (line.startsWith("time")) {
+ uptime = line.split(DELIMITER)[1];
+ } else if (line.startsWith("ram")) {
+ String[] ram = line.split(DELIMITER);
+ ramTotal = ram[1];
+ ramFree = ram[2];
+ ramUsed = ram[3];
+ } else if (line.contains(PROCESS_CAMERA_DAEMON)) {
+ pssCameraNative = line.split(DELIMITER)[4];
+ } else if (line.contains(PROCESS_CAMERA_APP)) {
+ pssCameraApp = line.split(DELIMITER)[4];
+ }
+ }
+ String printMsg = String.format(
+ "%s,%s,%s,%s,%s,%s", uptime, pssCameraNative, pssCameraApp,
+ ramTotal, ramFree, ramUsed);
+ writer.write(printMsg);
+ writer.newLine();
+ writer.flush();
+ } catch (IOException e) {
+ CLog.w("Failed to print meminfo to %s:", outputFile.getAbsolutePath());
+ CLog.e(e);
+ } finally {
+ StreamUtil.close(writer);
+ }
+ }
+
+ // TODO: Leverage AUPT to collect system logs (meminfo, ION allocations and processes/threads)
+ private class ThreadTrackerTimer {
+
+ // list all threads in a given process, remove the first header line, squeeze whitespaces,
+ // select thread name (in 14th column), then sort and group by its name.
+ // Examples:
+ // 3 SoundPoolThread
+ // 3 SoundPool
+ // 2 Camera Job Disp
+ // 1 pool-7-thread-1
+ // 1 pool-6-thread-1
+ // FIXME: Resolve the error "sh: syntax error: '|' unexpected" using the command below
+ // $ /system/bin/ps -t -p %s | tr -s ' ' | cut -d' ' -f13- | sort | uniq -c | sort -nr"
+ private static final String PS_COMMAND_FORMAT = "/system/bin/ps -t -p %s";
+ private static final String PGREP_COMMAND_FORMAT = "pgrep %s";
+ private static final int STATE_STOPPED = 0;
+ private static final int STATE_SCHEDULED = 1;
+ private static final int STATE_RUNNING = 2;
+
+ private int mState = STATE_STOPPED;
+ private Timer mTimer = new Timer(true); // run as a daemon thread
+ private long mDelayMs = 0;
+ private long mPeriodMs = 60 * 1000; // 60 sec
+ private File mOutputFile = null;
+
+ public ThreadTrackerTimer(long delayMs, long periodMs) {
+ mDelayMs = delayMs;
+ mPeriodMs = periodMs;
+ }
+
+ synchronized void start(TestIdentifier test) {
+ if (isRunning()) {
+ stop();
+ }
+ // Create an output file.
+ if (createOutputFile(test) == null) {
+ CLog.w("Stop collecting thread counts since log file not found.");
+ mState = STATE_STOPPED;
+ return; // No-op
+ }
+ mTimer.scheduleAtFixedRate(new TimerTask() {
+ @Override
+ public void run() {
+ mState = STATE_RUNNING;
+ dumpThreadCount(PS_COMMAND_FORMAT, getPid(PROCESS_CAMERA_APP), mOutputFile);
+ }
+ }, mDelayMs, mPeriodMs);
+ mState = STATE_SCHEDULED;
+ }
+
+ synchronized void stop() {
+ mState = STATE_STOPPED;
+ mTimer.cancel();
+ }
+
+ synchronized boolean isRunning() {
+ return (mState == STATE_RUNNING);
+ }
+
+ public File getOutputFile() {
+ return mOutputFile;
+ }
+
+ File createOutputFile(TestIdentifier test) {
+ try {
+ mOutputFile = FileUtil.createTempFile(
+ String.format("ps_%s", test.getTestName()), "txt");
+ new BufferedWriter(new FileWriter(mOutputFile, false)).close();
+ } catch (IOException e) {
+ CLog.w("Failed to create processes and threads file %s:",
+ mOutputFile.getAbsolutePath());
+ CLog.e(e);
+ return null;
+ }
+ return mOutputFile;
+ }
+
+ String getPid(String processName) {
+ String result = null;
+ try {
+ result = getDevice().executeShellCommand(String.format(PGREP_COMMAND_FORMAT,
+ processName));
+ } catch (DeviceNotAvailableException e) {
+ CLog.w("Failed to get pid %s:", processName);
+ CLog.e(e);
+ }
+ return result;
+ }
+
+ String getUptime() {
+ String uptime = null;
+ try {
+ // uptime will typically have a format like "5278.73 1866.80". Use the first one
+ // (which is wall-time)
+ uptime = getDevice().executeShellCommand("cat /proc/uptime").split(" ")[0];
+ Float.parseFloat(uptime);
+ } catch (NumberFormatException e) {
+ CLog.w("Failed to get valid uptime %s: %s", uptime, e);
+ } catch (DeviceNotAvailableException e) {
+ CLog.w("Failed to get valid uptime: %s", e);
+ }
+ return uptime;
+ }
+
+ void dumpThreadCount(String commandFormat, String pid, File outputFile) {
+ try {
+ if ("".equals(pid)) {
+ return;
+ }
+ String result = getDevice().executeShellCommand(String.format(commandFormat, pid));
+ String header = String.format("UPTIME: %s", getUptime());
+ BufferedWriter writer = new BufferedWriter(new FileWriter(outputFile, true));
+ writer.write(header);
+ writer.newLine();
+ writer.write(result);
+ writer.newLine();
+ writer.flush();
+ writer.close();
+ } catch (DeviceNotAvailableException | IOException e) {
+ CLog.w("Failed to dump thread count:");
+ CLog.e(e);
+ }
+ }
+ }
+
+ // TODO: Leverage AUPT to collect system logs (meminfo, ION allocations and
+ // processes/threads)
+ protected void dumpIonHeaps(ITestInvocationListener listener, String testClass) {
+ if (!shouldDumpIonHeap()) {
+ return; // No-op if option is not set.
+ }
+ try {
+ String result = getDevice().executeShellCommand(DUMP_ION_HEAPS_COMMAND);
+ if (!"".equals(result)) {
+ String fileName = String.format("ionheaps_%s_onEnd", testClass);
+ listener.testLog(fileName, LogDataType.TEXT,
+ new ByteArrayInputStreamSource(result.getBytes()));
+ }
+ } catch (DeviceNotAvailableException e) {
+ CLog.w("Failed to dump ION heaps:");
+ CLog.e(e);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ITestDevice getDevice() {
+ return mDevice;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setConfiguration(IConfiguration configuration) {
+ mConfiguration = configuration;
+ }
+
+ /**
+ * Get the {@link IRunUtil} instance to use.
+ * <p/>
+ * Exposed so unit tests can mock.
+ */
+ IRunUtil getRunUtil() {
+ return RunUtil.getDefault();
+ }
+
+ /**
+ * Get the duration of Camera test instrumentation in milliseconds.
+ *
+ * @return the duration of Camera instrumentation test until it is called.
+ */
+ public long getTestDurationMs() {
+ return System.currentTimeMillis() - mStartTimeMs;
+ }
+
+ public String getTestPackage() {
+ return mTestPackage;
+ }
+
+ public void setTestPackage(String testPackage) {
+ mTestPackage = testPackage;
+ }
+
+ public String getTestClass() {
+ return mTestClass;
+ }
+
+ public void setTestClass(String testClass) {
+ mTestClass = testClass;
+ }
+
+ public String getTestRunner() {
+ return mTestRunner;
+ }
+
+ public void setTestRunner(String testRunner) {
+ mTestRunner = testRunner;
+ }
+
+ public int getTestTimeoutMs() {
+ return mTestTimeoutMs;
+ }
+
+ public void setTestTimeoutMs(int testTimeoutMs) {
+ mTestTimeoutMs = testTimeoutMs;
+ }
+
+ public long getShellTimeoutMs() {
+ return mShellTimeoutMs;
+ }
+
+ public void setShellTimeoutMs(long shellTimeoutMs) {
+ mShellTimeoutMs = shellTimeoutMs;
+ }
+
+ public String getRuKey() {
+ return mRuKey;
+ }
+
+ public void setRuKey(String ruKey) {
+ mRuKey = ruKey;
+ }
+
+ public boolean shouldDumpMeminfo() {
+ return mDumpMeminfo;
+ }
+
+ public boolean shouldDumpIonHeap() {
+ return mDumpIonHeap;
+ }
+
+ public boolean shouldDumpThreadCount() {
+ return mDumpThreadCount;
+ }
+
+ public AbstractCollectingListener getCollectingListener() {
+ return mCollectingListener;
+ }
+
+ public void setLogcatOnFailure(boolean logcatOnFailure) {
+ mLogcatOnFailure = logcatOnFailure;
+ }
+
+ public int getIterationCount() {
+ return mIterations;
+ }
+
+ public Map<String, String> getInstrumentationArgMap() { return mInstrArgMap; }
+}
diff --git a/src/com/android/media/tests/MediaMemoryTest.java b/src/com/android/media/tests/MediaMemoryTest.java
new file mode 100644
index 0000000..bb09488
--- /dev/null
+++ b/src/com/android/media/tests/MediaMemoryTest.java
@@ -0,0 +1,280 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
+import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.BugreportCollector;
+import com.android.tradefed.result.BugreportCollector.Freq;
+import com.android.tradefed.result.BugreportCollector.Noun;
+import com.android.tradefed.result.BugreportCollector.Relation;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.StreamUtil;
+
+import org.junit.Assert;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Runs the Media memory test. This test will do various media actions ( ie.
+ * playback, recording and etc.) then capture the snapshot of mediaserver memory
+ * usage. The test summary is save to /sdcard/mediaMemOutput.txt
+ * <p/>
+ * Note that this test will not run properly unless /sdcard is mounted and
+ * writable.
+ */
+public class MediaMemoryTest implements IDeviceTest, IRemoteTest {
+
+ ITestDevice mTestDevice = null;
+
+ private static final String METRICS_RUN_NAME = "MediaMemoryLeak";
+
+ // Constants for running the tests
+ private static final String TEST_CLASS_NAME =
+ "com.android.mediaframeworktest.performance.MediaPlayerPerformance";
+ private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest";
+ private static final String TEST_RUNNER_NAME = ".MediaFrameworkPerfTestRunner";
+
+ private final String mOutputPaths[] = {"mediaMemOutput.txt","mediaProcmemOutput.txt"};
+
+ //Max test timeout - 4 hrs
+ private static final int MAX_TEST_TIMEOUT = 4 * 60 * 60 * 1000;
+
+ public Map<String, String> mPatternMap = new HashMap<>();
+ private static final Pattern TOTAL_MEM_DIFF_PATTERN =
+ Pattern.compile("^The total diff = (\\d+)");
+
+ @Option(name = "getHeapDump", description = "Collect the heap ")
+ private boolean mGetHeapDump = false;
+
+ @Option(name = "getProcMem", description = "Collect the procmem info ")
+ private boolean mGetProcMem = false;
+
+ @Option(name = "testName", description = "Test name to run. May be repeated.")
+ private Collection<String> mTests = new LinkedList<>();
+
+ public MediaMemoryTest() {
+ mPatternMap.put("testCameraPreviewMemoryUsage", "CameraPreview");
+ mPatternMap.put("testRecordAudioOnlyMemoryUsage", "AudioRecord");
+ mPatternMap.put("testH263VideoPlaybackMemoryUsage", "H263Playback");
+ mPatternMap.put("testRecordVideoAudioMemoryUsage", "H263RecordVideoAudio");
+ mPatternMap.put("testH263RecordVideoOnlyMemoryUsage", "H263RecordVideoOnly");
+ mPatternMap.put("testH264VideoPlaybackMemoryUsage", "H264Playback");
+ mPatternMap.put("testMpeg4RecordVideoOnlyMemoryUsage", "MPEG4RecordVideoOnly");
+ }
+
+
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ Assert.assertNotNull(mTestDevice);
+
+ IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME,
+ TEST_RUNNER_NAME, mTestDevice.getIDevice());
+ runner.setClassName(TEST_CLASS_NAME);
+ runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS);
+ if (mGetHeapDump) {
+ runner.addInstrumentationArg("get_heap_dump", "true");
+ }
+ if (mGetProcMem) {
+ runner.addInstrumentationArg("get_procmem", "true");
+ }
+
+ BugreportCollector bugListener = new BugreportCollector(listener,
+ mTestDevice);
+ bugListener.addPredicate(new BugreportCollector.Predicate(
+ Relation.AFTER, Freq.EACH, Noun.TESTRUN));
+
+ if (mTests.size() > 0) {
+ for (String testName : mTests) {
+ runner.setMethodName(TEST_CLASS_NAME, testName);
+ mTestDevice.runInstrumentationTests(runner, bugListener);
+ }
+ } else {
+ mTestDevice.runInstrumentationTests(runner, bugListener);
+ }
+ logOutputFiles(listener);
+ cleanResultFile();
+ }
+
+ /**
+ * Clean up the test result file from test run
+ */
+ private void cleanResultFile() throws DeviceNotAvailableException {
+ String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+ for(String outputPath : mOutputPaths){
+ mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, outputPath));
+ }
+ if (mGetHeapDump) {
+ mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, "*.dump"));
+ }
+ }
+
+ private void uploadHeapDumpFiles(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ // Pull and upload the heap dump output files.
+ InputStreamSource outputSource = null;
+ File outputFile = null;
+
+ String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+
+ String out = mTestDevice.executeShellCommand(String.format("ls %s/%s",
+ extStore, "*.dump"));
+ String heapOutputFiles[] = out.split("\n");
+
+ for (String heapFile : heapOutputFiles) {
+ try {
+ outputFile = mTestDevice.pullFile(heapFile.trim());
+ if (outputFile == null) {
+ continue;
+ }
+ outputSource = new FileInputStreamSource(outputFile);
+ listener.testLog(heapFile, LogDataType.TEXT, outputSource);
+ } finally {
+ FileUtil.deleteFile(outputFile);
+ StreamUtil.cancel(outputSource);
+ }
+ }
+ }
+
+ /**
+ * Pull the output files from the device, add it to the logs, and also parse
+ * out the relevant test metrics and report them.
+ */
+ private void logOutputFiles(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ File outputFile = null;
+ InputStreamSource outputSource = null;
+
+ if (mGetHeapDump) {
+ // Upload all the heap dump files.
+ uploadHeapDumpFiles(listener);
+ }
+ for(String outputPath : mOutputPaths){
+ try {
+ outputFile = mTestDevice.pullFileFromExternal(outputPath);
+
+ if (outputFile == null) {
+ return;
+ }
+
+ // Upload a verbatim copy of the output file
+ CLog.d("Sending %d byte file %s into the logosphere!",
+ outputFile.length(), outputFile);
+ outputSource = new FileInputStreamSource(outputFile);
+ listener.testLog(outputPath, LogDataType.TEXT, outputSource);
+
+ // Parse the output file to upload aggregated metrics
+ parseOutputFile(new FileInputStream(outputFile), listener);
+ } catch (IOException e) {
+ CLog.e("IOException while reading or parsing output file: %s",
+ e.getMessage());
+ } finally {
+ FileUtil.deleteFile(outputFile);
+ StreamUtil.cancel(outputSource);
+ }
+ }
+ }
+
+ /**
+ * Parse the relevant metrics from the Instrumentation test output file
+ */
+ private void parseOutputFile(InputStream dataStream,
+ ITestInvocationListener listener) {
+
+ Map<String, String> runMetrics = new HashMap<>();
+
+ // try to parse it
+ String contents;
+ try {
+ contents = StreamUtil.getStringFromStream(dataStream);
+ } catch (IOException e) {
+ CLog.e("Got IOException during test processing: %s",
+ e.getMessage());
+ return;
+ }
+
+ List<String> lines = Arrays.asList(contents.split("\n"));
+ ListIterator<String> lineIter = lines.listIterator();
+ String line;
+ while (lineIter.hasNext()) {
+ line = lineIter.next();
+ if (mPatternMap.containsKey(line)) {
+
+ String key = mPatternMap.get(line);
+ // Look for the total diff
+ while (lineIter.hasNext()) {
+ line = lineIter.next();
+ Matcher m = TOTAL_MEM_DIFF_PATTERN.matcher(line);
+ if (m.matches()) {
+ int result = Integer.parseInt(m.group(1));
+ runMetrics.put(key, Integer.toString(result));
+ break;
+ }
+ }
+ } else {
+ CLog.e("Got unmatched line: %s", line);
+ continue;
+ }
+ }
+ reportMetrics(listener, runMetrics);
+ }
+
+ /**
+ * Report run metrics by creating an empty test run to stick them in
+ * <p />
+ * Exposed for unit testing
+ */
+ void reportMetrics(ITestInvocationListener listener, Map<String, String> metrics) {
+ CLog.d("About to report metrics: %s", metrics);
+ listener.testRunStarted(METRICS_RUN_NAME, 0);
+ listener.testRunEnded(0, metrics);
+ }
+
+ @Override
+ public void setDevice(ITestDevice device) {
+ mTestDevice = device;
+ }
+
+ @Override
+ public ITestDevice getDevice() {
+ return mTestDevice;
+ }
+}
diff --git a/src/com/android/media/tests/MediaPlayerStressTest.java b/src/com/android/media/tests/MediaPlayerStressTest.java
new file mode 100644
index 0000000..8efab3e
--- /dev/null
+++ b/src/com/android/media/tests/MediaPlayerStressTest.java
@@ -0,0 +1,212 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.Log;
+import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
+import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.Option.Importance;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.result.BugreportCollector;
+import com.android.tradefed.result.BugreportCollector.Freq;
+import com.android.tradefed.result.BugreportCollector.Noun;
+import com.android.tradefed.result.BugreportCollector.Relation;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.RegexTrie;
+import com.android.tradefed.util.StreamUtil;
+
+import org.junit.Assert;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Runs the Media Player stress test. This test will play the video files under
+ * the /sdcard/samples folder and capture the video playback event statistics in
+ * a text file under /sdcard/PlaybackTestResult.txt
+ * <p/>
+ * Note that this test will not run properly unless /sdcard is mounted and
+ * writable.
+ */
+public class MediaPlayerStressTest implements IDeviceTest, IRemoteTest {
+ private static final String LOG_TAG = "MediaPlayerStress";
+
+ ITestDevice mTestDevice = null;
+ @Option(name = "test-class", importance = Importance.ALWAYS)
+ private String mTestClassName =
+ "com.android.mediaframeworktest.stress.MediaPlayerStressTest";
+ @Option(name = "metrics-name", importance = Importance.ALWAYS)
+ private String mMetricsRunName = "MediaPlayerStress";
+ @Option(name = "result-file", importance = Importance.ALWAYS)
+ private String mOutputPath = "PlaybackTestResult.txt";
+
+ //Max test timeout - 10 hrs
+ private static final int MAX_TEST_TIMEOUT = 10 * 60 * 60 * 1000;
+
+ // Constants for running the tests
+ private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest";
+ private static final String TEST_RUNNER_NAME = ".MediaPlayerStressTestRunner";
+
+ public RegexTrie<String> mPatternMap = new RegexTrie<>();
+
+ public MediaPlayerStressTest() {
+ mPatternMap.put("PlaybackPass", "^Total Complete: (\\d+)");
+ mPatternMap.put("PlaybackCrash", "^Total Error: (\\d+)");
+ mPatternMap.put("TrackLagging", "^Total Track Lagging: (\\d+)");
+ mPatternMap.put("BadInterleave", "^Total Bad Interleaving: (\\d+)");
+ mPatternMap.put("FailedToCompleteWithNoError",
+ "^Total Failed To Complete With No Error: (\\d+)");
+ }
+
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ Assert.assertNotNull(mTestDevice);
+ IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME,
+ TEST_RUNNER_NAME, mTestDevice.getIDevice());
+ runner.setClassName(mTestClassName);
+ runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS);
+
+ BugreportCollector bugListener = new BugreportCollector(listener,
+ mTestDevice);
+ bugListener.addPredicate(BugreportCollector.AFTER_FAILED_TESTCASES);
+ bugListener.setDescriptiveName("media_player_stress_test");
+ bugListener.addPredicate(new BugreportCollector.Predicate(
+ Relation.AFTER, Freq.EACH, Noun.TESTRUN));
+
+ mTestDevice.runInstrumentationTests(runner, bugListener);
+
+ logOutputFile(listener);
+ cleanResultFile();
+ }
+
+ /**
+ * Clean up the test result file from test run
+ */
+ private void cleanResultFile() throws DeviceNotAvailableException {
+ String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+ mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath));
+ }
+
+ /**
+ * Pull the output file from the device, add it to the logs, and also parse
+ * out the relevant test metrics and report them.
+ */
+ private void logOutputFile(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ File outputFile = null;
+ InputStreamSource outputSource = null;
+ try {
+ outputFile = mTestDevice.pullFileFromExternal(mOutputPath);
+
+ if (outputFile == null) {
+ return;
+ }
+
+ // Upload a verbatim copy of the output file
+ Log.d(LOG_TAG, String.format("Sending %d byte file %s into the logosphere!",
+ outputFile.length(), outputFile));
+ outputSource = new FileInputStreamSource(outputFile);
+ listener.testLog(mOutputPath, LogDataType.TEXT, outputSource);
+ // Parse the output file to upload aggregated metrics
+ parseOutputFile(new FileInputStream(outputFile), listener);
+ } catch (IOException e) {
+ Log.e(LOG_TAG, String.format(
+ "IOException while reading or parsing output file: %s", e));
+ } finally {
+ FileUtil.deleteFile(outputFile);
+ StreamUtil.cancel(outputSource);
+ }
+ }
+
+ /**
+ * Parse the relevant metrics from the Instrumentation test output file
+ */
+ private void parseOutputFile(InputStream dataStream,
+ ITestInvocationListener listener) {
+ Map<String, String> runMetrics = new HashMap<>();
+
+ // try to parse it
+ String contents;
+ try {
+ contents = StreamUtil.getStringFromStream(dataStream);
+ } catch (IOException e) {
+ Log.e(LOG_TAG, String.format(
+ "Got IOException during test processing: %s", e));
+ return;
+ }
+
+ List<String> lines = Arrays.asList(contents.split("\n"));
+ ListIterator<String> lineIter = lines.listIterator();
+ String line;
+ while (lineIter.hasNext()) {
+ line = lineIter.next();
+ List<List<String>> capture = new ArrayList<>(1);
+ String key = mPatternMap.retrieve(capture, line);
+ if (key != null) {
+ Log.d(LOG_TAG, String.format("Got '%s' and captures '%s'",
+ key, capture.toString()));
+ } else if (line.isEmpty()) {
+ // ignore
+ continue;
+ } else {
+ Log.d(LOG_TAG, String.format("Got unmatched line: %s", line));
+ continue;
+ }
+ runMetrics.put(key, capture.get(0).get(0));
+ }
+ reportMetrics(listener, runMetrics);
+ }
+
+ /**
+ * Report run metrics by creating an empty test run to stick them in
+ * <p />
+ * Exposed for unit testing
+ */
+ void reportMetrics(ITestInvocationListener listener, Map<String, String> metrics) {
+ Log.d(LOG_TAG, String.format("About to report metrics: %s", metrics));
+ listener.testRunStarted(mMetricsRunName, 0);
+ listener.testRunEnded(0, metrics);
+ }
+
+ @Override
+ public void setDevice(ITestDevice device) {
+ mTestDevice = device;
+ }
+
+ @Override
+ public ITestDevice getDevice() {
+ return mTestDevice;
+ }
+}
diff --git a/src/com/android/media/tests/MediaResultReporter.java b/src/com/android/media/tests/MediaResultReporter.java
new file mode 100644
index 0000000..0d348c3
--- /dev/null
+++ b/src/com/android/media/tests/MediaResultReporter.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.Log;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.result.EmailResultReporter;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.result.TestSummary;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.List;
+
+/**
+ * Media reporter that send the test summary through email.
+ */
+public class MediaResultReporter extends EmailResultReporter {
+
+ private static final String LOG_TAG = "MediaResultReporter";
+
+ public StringBuilder mEmailBodyBuilder;
+ private String mSummaryUrl = "";
+
+ @Option(name = "log-name", description = "Name of the report that attach to email")
+ private String mLogName = null;
+
+ public MediaResultReporter() {
+ mEmailBodyBuilder = new StringBuilder();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String generateEmailBody() {
+ return mEmailBodyBuilder.toString();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void putSummary(List<TestSummary> summaries) {
+ // Get the summary url
+ if (summaries.isEmpty()) {
+ return;
+ }
+ mSummaryUrl = summaries.get(0).getSummary().getString();
+ mEmailBodyBuilder.append(mSummaryUrl);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void testLog(String dataName, LogDataType dataType, InputStreamSource dataStream) {
+
+ char[] buf = new char[2048];
+
+ try {
+ //Attached the test summary to the email body
+ if (mLogName.equals(dataName)) {
+ Reader r = new InputStreamReader(dataStream.createInputStream());
+ while (true) {
+ int n = r.read(buf);
+ if (n < 0) {
+ break;
+ }
+ mEmailBodyBuilder.append(buf, 0, n);
+ }
+ mEmailBodyBuilder.append('\n');
+ }
+ } catch (IOException e) {
+ Log.w(LOG_TAG, String.format("Exception while parsing %s: %s", dataName, e));
+ }
+ }
+}
diff --git a/src/com/android/media/tests/MediaStressTest.java b/src/com/android/media/tests/MediaStressTest.java
new file mode 100644
index 0000000..189dd60
--- /dev/null
+++ b/src/com/android/media/tests/MediaStressTest.java
@@ -0,0 +1,225 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
+import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.StreamUtil;
+
+import org.junit.Assert;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Runs the Media stress testcases.
+ * FIXME: more details
+ * <p/>
+ * Note that this test will not run properly unless /sdcard is mounted and writable.
+ */
+public class MediaStressTest implements IDeviceTest, IRemoteTest {
+
+ ITestDevice mTestDevice = null;
+ private static final String METRICS_RUN_NAME = "VideoRecordingStress";
+
+ //Max test timeout - 2 hrs
+ private static final int MAX_TEST_TIMEOUT = 2 * 60 * 60 * 1000;
+
+ // Constants for running the tests
+ private static final String TEST_CLASS_NAME =
+ "com.android.mediaframeworktest.stress.MediaRecorderStressTest";
+ private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest";
+ private static final String TEST_RUNNER_NAME = ".MediaRecorderStressTestRunner";
+
+ // Constants for parsing the output file
+ private static final Pattern EXPECTED_LOOP_COUNT_PATTERN =
+ Pattern.compile("Total number of loops:\\s*(\\d+)");
+ private static final Pattern ACTUAL_LOOP_COUNT_PATTERN =
+ Pattern.compile("No of loop:.*,\\s*(\\d+)\\s*");
+ private static final String OUTPUT_PATH = "mediaStressOutput.txt";
+
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ Assert.assertNotNull(mTestDevice);
+
+ IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME,
+ TEST_RUNNER_NAME, mTestDevice.getIDevice());
+ runner.setClassName(TEST_CLASS_NAME);
+ runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS);
+
+ cleanTmpFiles();
+ mTestDevice.runInstrumentationTests(runner, listener);
+ logOutputFile(listener);
+ cleanTmpFiles();
+ }
+
+ /**
+ * Clean up temp files from test runs
+ */
+ private void cleanTmpFiles() throws DeviceNotAvailableException {
+ String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+ mTestDevice.executeShellCommand(String.format("rm %s/temp*.3gp", extStore));
+ mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, OUTPUT_PATH));
+ }
+
+ /**
+ * Pull the output file from the device, add it to the logs, and also parse out the relevant
+ * test metrics and report them.
+ */
+ private void logOutputFile(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ File outputFile = null;
+ InputStreamSource outputSource = null;
+ try {
+ outputFile = mTestDevice.pullFileFromExternal(OUTPUT_PATH);
+
+ if (outputFile == null) {
+ return;
+ }
+
+ CLog.d("Sending %d byte file %s into the logosphere!", outputFile.length(), outputFile);
+ outputSource = new FileInputStreamSource(outputFile);
+ listener.testLog(OUTPUT_PATH, LogDataType.TEXT, outputSource);
+ parseOutputFile(outputFile, listener);
+ } finally {
+ FileUtil.deleteFile(outputFile);
+ StreamUtil.cancel(outputSource);
+ }
+ }
+
+ /**
+ * Parse the relevant metrics from the Instrumentation test output file
+ */
+ private void parseOutputFile(File outputFile, ITestInvocationListener listener) {
+ Map<String, String> runMetrics = new HashMap<>();
+ Map<String, String> stanzaKeyMap = new HashMap<>();
+ stanzaKeyMap.put("testStressRecordVideoAndPlayback1080P", "VideoRecordPlayback1080P");
+ stanzaKeyMap.put("testStressRecordVideoAndPlayback720P", "VideoRecordPlayback720P");
+ stanzaKeyMap.put("testStressRecordVideoAndPlayback480P", "VideoRecordPlayback480P");
+ stanzaKeyMap.put("testStressTimeLapse", "TimeLapseRecord");
+
+ // try to parse it
+ String contents;
+ try {
+ InputStream dataStream = new FileInputStream(outputFile);
+ contents = StreamUtil.getStringFromStream(dataStream);
+ } catch (IOException e) {
+ CLog.e("IOException while parsing the output file:");
+ CLog.e(e);
+ return;
+ }
+
+ List<String> lines = Arrays.asList(contents.split("\n"));
+ ListIterator<String> lineIter = lines.listIterator();
+ String line;
+ while (lineIter.hasNext()) {
+ line = lineIter.next();
+ String key = null;
+
+ if (stanzaKeyMap.containsKey(line)) {
+ key = stanzaKeyMap.get(line);
+ } else {
+ CLog.d("Got unexpected line: %s", line);
+ continue;
+ }
+
+ Integer countExpected = getIntFromOutput(lineIter, EXPECTED_LOOP_COUNT_PATTERN);
+ Integer countActual = getIntFromOutput(lineIter, ACTUAL_LOOP_COUNT_PATTERN);
+ int value = coalesceLoopCounts(countActual, countExpected);
+ runMetrics.put(key, Integer.toString(value));
+ }
+
+ reportMetrics(listener, runMetrics);
+ }
+
+ /**
+ * Report run metrics by creating an empty test run to stick them in
+ * <p />
+ * Exposed for unit testing
+ */
+ void reportMetrics(ITestInvocationListener listener, Map<String, String> metrics) {
+ // Create an empty testRun to report the parsed runMetrics
+ CLog.d("About to report metrics: %s", metrics);
+ listener.testRunStarted(METRICS_RUN_NAME, 0);
+ listener.testRunEnded(0, metrics);
+ }
+
+ /**
+ * Use the provided {@link Pattern} to parse a number out of the output file
+ */
+ private Integer getIntFromOutput(ListIterator<String> lineIter, Pattern numPattern) {
+ Integer retval = null;
+ String line = null;
+ if (lineIter.hasNext()) {
+ line = lineIter.next();
+ Matcher m = numPattern.matcher(line);
+ if (m.matches()) {
+ retval = Integer.parseInt(m.group(1));
+ } else {
+ CLog.e("Couldn't match pattern %s against line '%s'", numPattern, line);
+ }
+ } else {
+ CLog.e("Encounted EOF while trying to match pattern %s", numPattern);
+ }
+
+ return retval;
+ }
+
+ /**
+ * Given an actual and an expected iteration count, determine a single metric to report.
+ */
+ private int coalesceLoopCounts(Integer actual, Integer expected) {
+ if (expected == null || expected <= 0) {
+ return -1;
+ } else if (actual == null) {
+ return expected;
+ } else {
+ return actual;
+ }
+ }
+
+ @Override
+ public void setDevice(ITestDevice device) {
+ mTestDevice = device;
+ }
+
+ @Override
+ public ITestDevice getDevice() {
+ return mTestDevice;
+ }
+}
diff --git a/src/com/android/media/tests/MediaTest.java b/src/com/android/media/tests/MediaTest.java
new file mode 100644
index 0000000..473f989
--- /dev/null
+++ b/src/com/android/media/tests/MediaTest.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import junit.framework.TestCase;
+
+import java.util.regex.Matcher;
+
+
+public class MediaTest extends TestCase {
+ private String mPositiveOutput = "testStressAddRemoveEffects diff : 472";
+ private String mNegativeOutput = "testStressAddRemoveEffects diff : -123";
+
+ private String mExpectedTestCaseName = "testStressAddRemoveEffects";
+ private String mExpectedPositiveOut = "472";
+ private String mExpectedNegativeOut = "-123";
+
+ public void testVideoEditorPositiveOut() {
+ Matcher m = VideoEditingMemoryTest.TOTAL_MEM_DIFF_PATTERN.matcher(mPositiveOutput);
+ assertTrue(m.matches());
+ assertEquals("Parse test case name", mExpectedTestCaseName, m.group(1));
+ assertEquals("Paser positive out", mExpectedPositiveOut, m.group(2));
+ }
+
+ public void testVideoEditorNegativeOut() {
+ Matcher m = VideoEditingMemoryTest.TOTAL_MEM_DIFF_PATTERN.matcher(mNegativeOutput);
+ assertTrue(m.matches());
+ assertEquals("Parse test case name", mExpectedTestCaseName, m.group(1));
+ assertEquals("Paser negative out", mExpectedNegativeOut, m.group(2));
+ }
+}
diff --git a/src/com/android/media/tests/PanoramaBenchMarkTest.java b/src/com/android/media/tests/PanoramaBenchMarkTest.java
new file mode 100644
index 0000000..355bbb0
--- /dev/null
+++ b/src/com/android/media/tests/PanoramaBenchMarkTest.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.IDevice;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+
+import org.junit.Assert;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Standalone panoramic photo processing benchmark test.
+ */
+public class PanoramaBenchMarkTest implements IDeviceTest, IRemoteTest {
+
+ private ITestDevice mTestDevice = null;
+
+ private static final Pattern ELAPSED_TIME_PATTERN =
+ Pattern.compile("(Total elapsed time:)\\s+(\\d+\\.\\d*)\\s+(seconds)");
+
+ private static final String PANORAMA_TEST_KEY = "PanoramaElapsedTime";
+ private static final String TEST_TAG = "CameraLatency";
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
+ Assert.assertNotNull(mTestDevice);
+
+ String dataStore = mTestDevice.getMountPoint(IDevice.MNT_DATA);
+ String externalStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+
+ mTestDevice.executeShellCommand(String.format("chmod 777 %s/local/tmp/panorama_bench",
+ dataStore));
+
+ String shellOutput = mTestDevice.executeShellCommand(
+ String.format("%s/local/tmp/panorama_bench %s/panorama_input/test %s/panorama.ppm",
+ dataStore, externalStore, externalStore));
+
+ String[] lines = shellOutput.split("\n");
+
+ Map<String, String> metrics = new HashMap<String, String>();
+ for (String line : lines) {
+ Matcher m = ELAPSED_TIME_PATTERN.matcher(line.trim());
+ if (m.matches()) {
+ CLog.d(String.format("Found elapsed time \"%s seconds\" from line %s",
+ m.group(2), line));
+ metrics.put(PANORAMA_TEST_KEY, m.group(2));
+ break;
+ } else {
+ CLog.d(String.format("Unabled to find elapsed time from line: %s", line));
+ }
+ }
+
+ reportMetrics(listener, TEST_TAG, metrics);
+ cleanupDevice();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mTestDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ITestDevice getDevice() {
+ return mTestDevice;
+ }
+
+ /**
+ * Removes image files used to mock panorama stitching.
+ *
+ * @throws DeviceNotAvailableException If the device is unavailable or
+ * something happened while deleting files
+ */
+ private void cleanupDevice() throws DeviceNotAvailableException {
+ String externalStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+ mTestDevice.executeShellCommand(String.format("rm -r %s/panorama_input", externalStore));
+ }
+
+ /**
+ * Report run metrics by creating an empty test run to stick them in.
+ *
+ * @param listener The {@link ITestInvocationListener} of test results
+ * @param runName The test name
+ * @param metrics The {@link Map} that contains metrics for the given test
+ */
+ private void reportMetrics(ITestInvocationListener listener, String runName,
+ Map<String, String> metrics) {
+ InputStreamSource bugreport = mTestDevice.getBugreport();
+ listener.testLog("bugreport", LogDataType.BUGREPORT, bugreport);
+ bugreport.cancel();
+
+ CLog.d(String.format("About to report metrics: %s", metrics));
+ listener.testRunStarted(runName, 0);
+ listener.testRunEnded(0, metrics);
+ }
+}
diff --git a/src/com/android/media/tests/TestRunHelper.java b/src/com/android/media/tests/TestRunHelper.java
new file mode 100644
index 0000000..f752cf3
--- /dev/null
+++ b/src/com/android/media/tests/TestRunHelper.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.media.tests;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** Generic helper class for tests */
+public class TestRunHelper {
+
+ private long mTestStartTime = -1;
+ private long mTestStopTime = -1;
+ private ITestInvocationListener mListener;
+ private TestIdentifier mTestId;
+
+ public TestRunHelper(ITestInvocationListener listener, TestIdentifier testId) {
+ mListener = listener;
+ mTestId = testId;
+ }
+
+ public long getTotalTestTime() {
+ return mTestStopTime - mTestStartTime;
+ }
+
+ public void reportFailure(String errMsg) {
+ CLog.e(errMsg);
+ mListener.testFailed(mTestId, errMsg);
+ mListener.testEnded(mTestId, new HashMap<String, String>());
+ mListener.testRunFailed(errMsg);
+ }
+
+ /** @param resultDictionary */
+ public void endTest(Map<String, String> resultDictionary) {
+ mTestStopTime = System.currentTimeMillis();
+ mListener.testEnded(mTestId, resultDictionary);
+ mListener.testRunEnded(getTotalTestTime(), resultDictionary);
+ }
+
+ public void startTest(int numberOfTests) {
+ mListener.testRunStarted(mTestId.getTestName(), numberOfTests);
+ mListener.testStarted(mTestId);
+ mTestStartTime = System.currentTimeMillis();
+ }
+}
diff --git a/src/com/android/media/tests/VideoEditingMemoryTest.java b/src/com/android/media/tests/VideoEditingMemoryTest.java
new file mode 100644
index 0000000..bb58df2
--- /dev/null
+++ b/src/com/android/media/tests/VideoEditingMemoryTest.java
@@ -0,0 +1,268 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.Log;
+import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
+import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.result.BugreportCollector;
+import com.android.tradefed.result.BugreportCollector.Freq;
+import com.android.tradefed.result.BugreportCollector.Noun;
+import com.android.tradefed.result.BugreportCollector.Relation;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.StreamUtil;
+
+import org.junit.Assert;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Runs the Video Editing Framework Memory Tests. This test exercise the basic
+ * functionality of video editing test and capture the memory usage. The memory
+ * usage test out is saved in /sdcard/VideoEditorStressMemOutput.txt and
+ * VideoEditorMediaServerMemoryLog.txt.
+ * <p/>
+ * Note that this test will not run properly unless /sdcard is mounted and
+ * writable.
+ */
+public class VideoEditingMemoryTest implements IDeviceTest, IRemoteTest {
+ private static final String LOG_TAG = "VideoEditorMemoryTest";
+
+ ITestDevice mTestDevice = null;
+
+ // Constants for running the tests
+ private static final String TEST_CLASS_NAME =
+ "com.android.mediaframeworktest.stress.VideoEditorStressTest";
+ private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest";
+ private static final String TEST_RUNNER_NAME = ".MediaPlayerStressTestRunner";
+
+ //Max test timeout - 3 hrs
+ private static final int MAX_TEST_TIMEOUT = 3 * 60 * 60 * 1000;
+
+ /*
+ * Pattern to find the test case name and test result.
+ * Example of a matching line:
+ * testStressAddRemoveEffects total diff = 0
+ * The first string 'testStressAddRemoveEffects' represent the dashboard key and
+ * the last string represent the test result.
+ */
+ public static final Pattern TOTAL_MEM_DIFF_PATTERN =
+ Pattern.compile("(.+?)\\s.*diff.*\\s(-?\\d+)");
+
+ public Map<String, String> mRunMetrics = new HashMap<>();
+ public Map<String, String> mKeyMap = new HashMap<>();
+
+ @Option(name = "getHeapDump", description = "Collect the heap")
+ private boolean mGetHeapDump = false;
+
+ public VideoEditingMemoryTest() {
+ mKeyMap.put("VideoEditorStressMemOutput.txt", "VideoEditorMemory");
+ mKeyMap.put("VideoEditorMediaServerMemoryLog.txt",
+ "VideoEditorMemoryMediaServer");
+ }
+
+
+ @Override
+ public void run(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ Assert.assertNotNull(mTestDevice);
+
+ IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(
+ TEST_PACKAGE_NAME, TEST_RUNNER_NAME, mTestDevice.getIDevice());
+ runner.setClassName(TEST_CLASS_NAME);
+ runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS);
+ if (mGetHeapDump) {
+ runner.addInstrumentationArg("get_heap_dump", "getNativeHeap");
+ }
+
+ BugreportCollector bugListener = new BugreportCollector(listener,
+ mTestDevice);
+ bugListener.addPredicate(new BugreportCollector.Predicate(
+ Relation.AFTER, Freq.EACH, Noun.TESTRUN));
+
+ mTestDevice.runInstrumentationTests(runner, bugListener);
+
+ logOutputFiles(listener);
+ cleanResultFile();
+ }
+
+ /**
+ * Clean up the test result file from test run
+ */
+ private void cleanResultFile() throws DeviceNotAvailableException {
+ String extStore =
+ mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+ for(String outFile : mKeyMap.keySet()) {
+ mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore,
+ outFile));
+ }
+ if (mGetHeapDump) {
+ mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore,
+ "*.dump"));
+ }
+ }
+
+ private void uploadHeapDumpFiles(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ // Pull and upload the heap dump output files.
+ InputStreamSource outputSource = null;
+ File outputFile = null;
+
+ String extStore =
+ mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+
+ String out = mTestDevice.executeShellCommand(String.format("ls %s/%s",
+ extStore, "*.dump"));
+ String heapOutputFiles[] = out.split("\n");
+
+ for (String heapOutputFile : heapOutputFiles) {
+ try {
+ outputFile = mTestDevice.pullFile(heapOutputFile.trim());
+ if (outputFile == null) {
+ continue;
+ }
+ outputSource = new FileInputStreamSource(outputFile);
+ listener.testLog(heapOutputFile, LogDataType.TEXT, outputSource);
+ } finally {
+ FileUtil.deleteFile(outputFile);
+ StreamUtil.cancel(outputSource);
+ }
+ }
+ }
+
+ /**
+ * Pull the output files from the device, add it to the logs, and also parse
+ * out the relevant test metrics and report them.
+ */
+ private void logOutputFiles(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ File outputFile = null;
+ InputStreamSource outputSource = null;
+
+ if (mGetHeapDump) {
+ // Upload all the heap dump files.
+ uploadHeapDumpFiles(listener);
+ }
+ for (String resultFile : mKeyMap.keySet()) {
+ try {
+ outputFile = mTestDevice.pullFileFromExternal(resultFile);
+
+ if (outputFile == null) {
+ return;
+ }
+
+ // Upload a verbatim copy of the output file
+ Log.d(LOG_TAG, String.format(
+ "Sending %d byte file %s into the logosphere!",
+ outputFile.length(), outputFile));
+ outputSource = new FileInputStreamSource(outputFile);
+ listener.testLog(resultFile, LogDataType.TEXT, outputSource);
+
+ // Parse the output file to upload aggregated metrics
+ parseOutputFile(new FileInputStream(outputFile), listener, resultFile);
+ } catch (IOException e) {
+ Log.e(
+ LOG_TAG,
+ String.format("IOException while reading or parsing output file: %s", e));
+ } finally {
+ FileUtil.deleteFile(outputFile);
+ StreamUtil.cancel(outputSource);
+ }
+ }
+ }
+
+ /**
+ * Parse the relevant metrics from the Instrumentation test output file
+ */
+ private void parseOutputFile(InputStream dataStream,
+ ITestInvocationListener listener, String outputFile) {
+
+ // try to parse it
+ String contents;
+ try {
+ contents = StreamUtil.getStringFromStream(dataStream);
+ } catch (IOException e) {
+ Log.e(LOG_TAG, String.format(
+ "Got IOException during test processing: %s", e));
+ return;
+ }
+
+ List<String> lines = Arrays.asList(contents.split("\n"));
+ ListIterator<String> lineIter = lines.listIterator();
+
+ String line;
+ String key;
+ String memOut;
+
+ while (lineIter.hasNext()){
+ line = lineIter.next();
+
+ // Look for the total diff
+ Matcher m = TOTAL_MEM_DIFF_PATTERN.matcher(line);
+ if (m.matches()){
+ //First group match with the test key name.
+ key = m.group(1);
+ //Second group match witht the test result.
+ memOut = m.group(2);
+ mRunMetrics.put(key, memOut);
+ }
+ }
+ reportMetrics(listener, outputFile);
+ }
+
+ /**
+ * Report run metrics by creating an empty test run to stick them in
+ * <p />
+ * Exposed for unit testing
+ */
+ void reportMetrics(ITestInvocationListener listener, String outputFile) {
+ Log.d(LOG_TAG, String.format("About to report metrics: %s", mRunMetrics));
+ listener.testRunStarted(mKeyMap.get(outputFile), 0);
+ listener.testRunEnded(0, mRunMetrics);
+ }
+
+ @Override
+ public void setDevice(ITestDevice device) {
+ mTestDevice = device;
+ }
+
+ @Override
+ public ITestDevice getDevice() {
+ return mTestDevice;
+ }
+}
diff --git a/src/com/android/media/tests/VideoEditingPerformanceTest.java b/src/com/android/media/tests/VideoEditingPerformanceTest.java
new file mode 100644
index 0000000..046403e
--- /dev/null
+++ b/src/com/android/media/tests/VideoEditingPerformanceTest.java
@@ -0,0 +1,257 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.IDevice;
+import com.android.ddmlib.Log;
+import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
+import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.result.BugreportCollector;
+import com.android.tradefed.result.BugreportCollector.Freq;
+import com.android.tradefed.result.BugreportCollector.Noun;
+import com.android.tradefed.result.BugreportCollector.Relation;
+import com.android.tradefed.result.FileInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.InputStreamSource;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.FileUtil;
+import com.android.tradefed.util.RegexTrie;
+import com.android.tradefed.util.StreamUtil;
+
+import org.junit.Assert;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Runs the Video Editing Framework Performance Test.The performance test result
+ * is saved in /sdcard/VideoEditorPerformance.txt
+ * <p/>
+ * Note that this test will not run properly unless /sdcard is mounted and
+ * writable.
+ */
+public class VideoEditingPerformanceTest implements IDeviceTest, IRemoteTest {
+ private static final String LOG_TAG = "VideoEditingPerformanceTest";
+
+ ITestDevice mTestDevice = null;
+
+ private static final String METRICS_RUN_NAME = "VideoEditor";
+
+ //Max test timeout - 3 hrs
+ private static final int MAX_TEST_TIMEOUT = 3 * 60 * 60 * 1000;
+
+ // Constants for running the tests
+ private static final String TEST_CLASS_NAME =
+ "com.android.mediaframeworktest.performance.VideoEditorPerformance";
+ private static final String TEST_PACKAGE_NAME = "com.android.mediaframeworktest";
+ private static final String TEST_RUNNER_NAME = ".MediaFrameworkPerfTestRunner";
+
+ private static final String OUTPUT_PATH = "VideoEditorPerformance.txt";
+
+ private final RegexTrie<String> mPatternMap = new RegexTrie<>();
+
+ public VideoEditingPerformanceTest() {
+ mPatternMap.put("ImageItemCreate",
+ "^.*Time taken to Create Media Image Item :(\\d+)");
+ mPatternMap.put("mageItemAdd",
+ "^.*Time taken to add Media Image Item :(\\d+)");
+ mPatternMap.put("ImageItemRemove",
+ "^.*Time taken to remove Media Image Item :(\\d+)");
+ mPatternMap.put("ImageItemCreate640x480",
+ "^.*Time taken to Create Media Image Item.*640x480.*:(\\d+)");
+ mPatternMap.put("ImageItemAdd640x480",
+ "^.*Time taken to add Media Image Item.*640x480.*:(\\d+)");
+ mPatternMap.put("ImageItemRemove640x480",
+ "^.*Time taken to remove Media Image Item.*640x480.*:(\\d+)");
+ mPatternMap.put("CrossFadeTransitionCreate",
+ "^.*Time taken to Create CrossFade Transition :(\\d+)");
+ mPatternMap.put("CrossFadeTransitionAdd",
+ "^.*Time taken to add CrossFade Transition :(\\d+)");
+ mPatternMap.put("CrossFadeTransitionRemove",
+ "^.*Time taken to remove CrossFade Transition :(\\d+)");
+ mPatternMap.put("VideoItemCreate",
+ "^.*Time taken to Create Media Video Item :(\\d+)");
+ mPatternMap.put("VideoItemAdd",
+ "^.*Time taken to Add Media Video Item :(\\d+)");
+ mPatternMap.put("VideoItemRemove",
+ "^.*Time taken to remove Media Video Item :(\\d+)");
+ mPatternMap.put("EffectOverlappingTransition",
+ "^.*Time taken to testPerformanceEffectOverlappingTransition :(\\d+.\\d+)");
+ mPatternMap.put("ExportStoryboard",
+ "^.*Time taken to do ONE export of storyboard duration 69000 is :(\\d+)");
+ mPatternMap.put("PreviewWithTransition",
+ "^.*Time taken to Generate Preview with transition :(\\d+.\\d+)");
+ mPatternMap.put("OverlayCreate",
+ "^.*Time taken to add & create Overlay :(\\d+)");
+ mPatternMap.put("OverlayRemove",
+ "^.*Time taken to remove Overlay :(\\d+)");
+ mPatternMap.put("GetVideoThumbnails",
+ "^.*Duration taken to get Video Thumbnails :(\\d+)");
+ mPatternMap.put("TransitionWithEffectOverlapping",
+ "^.*Time taken to TransitionWithEffectOverlapping :(\\d+.\\d+)");
+ mPatternMap.put("MediaPropertiesGet",
+ "^.*Time taken to get Media Properties :(\\d+)");
+ mPatternMap.put("AACLCAdd",
+ "^.*Time taken for 1st Audio Track.*AACLC.*:(\\d+)");
+ mPatternMap.put("AMRNBAdd",
+ "^.*Time taken for 2nd Audio Track.*AMRNB.*:(\\d+)");
+ mPatternMap.put("KenBurnGeneration",
+ "^.*Time taken to Generate KenBurn Effect :(\\d+.\\d+)");
+ mPatternMap.put("ThumbnailsGeneration",
+ "^.*Time taken Thumbnail generation :(\\d+.\\d+)");
+ mPatternMap.put("H264ThumbnailGeneration",
+ "^.*Time taken for Thumbnail generation :(\\d+.\\d+)");
+ }
+
+ @Override
+ public void run(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ Assert.assertNotNull(mTestDevice);
+
+ IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(
+ TEST_PACKAGE_NAME, TEST_RUNNER_NAME, mTestDevice.getIDevice());
+ runner.setClassName(TEST_CLASS_NAME);
+ runner.setMaxTimeToOutputResponse(MAX_TEST_TIMEOUT, TimeUnit.MILLISECONDS);
+
+ BugreportCollector bugListener = new BugreportCollector(listener,
+ mTestDevice);
+ bugListener.addPredicate(new BugreportCollector.Predicate(
+ Relation.AFTER, Freq.EACH, Noun.TESTRUN));
+
+ mTestDevice.runInstrumentationTests(runner, bugListener);
+
+ logOutputFiles(listener);
+ cleanResultFile();
+ }
+
+ /**
+ * Clean up the test result file from test run
+ */
+ private void cleanResultFile() throws DeviceNotAvailableException {
+ String extStore =
+ mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE);
+ mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, OUTPUT_PATH));
+ }
+
+ /**
+ * Pull the output files from the device, add it to the logs, and also parse
+ * out the relevant test metrics and report them.
+ */
+ private void logOutputFiles(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ File outputFile = null;
+ InputStreamSource outputSource = null;
+ try {
+ outputFile = mTestDevice.pullFileFromExternal(OUTPUT_PATH);
+
+ if (outputFile == null) {
+ return;
+ }
+
+ // Upload a verbatim copy of the output file
+ Log.d(LOG_TAG, String.format(
+ "Sending %d byte file %s into the logosphere!",
+ outputFile.length(), outputFile));
+ outputSource = new FileInputStreamSource(outputFile);
+ listener.testLog(OUTPUT_PATH, LogDataType.TEXT, outputSource);
+
+ // Parse the output file to upload aggregated metrics
+ parseOutputFile(new FileInputStream(outputFile), listener);
+ } catch (IOException e) {
+ Log.e(LOG_TAG, String.format(
+ "IOException while reading or parsing output file: %s", e));
+ } finally {
+ FileUtil.deleteFile(outputFile);
+ StreamUtil.cancel(outputSource);
+ }
+ }
+
+ /**
+ * Parse the relevant metrics from the Instrumentation test output file
+ */
+ private void parseOutputFile(InputStream dataStream,
+ ITestInvocationListener listener) {
+
+ Map<String, String> runMetrics = new HashMap<>();
+
+ // try to parse it
+ String contents;
+ try {
+ contents = StreamUtil.getStringFromStream(dataStream);
+ } catch (IOException e) {
+ Log.e(LOG_TAG, String.format(
+ "Got IOException during test processing: %s", e));
+ return;
+ }
+
+ List<String> lines = Arrays.asList(contents.split("\n"));
+ ListIterator<String> lineIter = lines.listIterator();
+ String line;
+ while (lineIter.hasNext()) {
+ line = lineIter.next();
+ List<List<String>> capture = new ArrayList<>(1);
+ String key = mPatternMap.retrieve(capture, line);
+ if (key != null) {
+ Log.d(LOG_TAG, String.format("Got '%s' and captures '%s'", key,
+ capture.toString()));
+ } else if (line.isEmpty()) {
+ // ignore
+ continue;
+ } else {
+ Log.e(LOG_TAG, String.format("Got unmatched line: %s", line));
+ continue;
+ }
+ runMetrics.put(key, capture.get(0).get(0));
+ }
+ reportMetrics(listener, runMetrics);
+ }
+
+ /**
+ * Report run metrics by creating an empty test run to stick them in
+ * <p />
+ * Exposed for unit testing
+ */
+ void reportMetrics(ITestInvocationListener listener,
+ Map<String, String> metrics) {
+ Log.d(LOG_TAG, String.format("About to report metrics: %s", metrics));
+ listener.testRunStarted(METRICS_RUN_NAME, 0);
+ listener.testRunEnded(0, metrics);
+ }
+
+ @Override
+ public void setDevice(ITestDevice device) {
+ mTestDevice = device;
+ }
+
+ @Override
+ public ITestDevice getDevice() {
+ return mTestDevice;
+ }
+}
diff --git a/src/com/android/media/tests/VideoMultimeterRunner.java b/src/com/android/media/tests/VideoMultimeterRunner.java
new file mode 100644
index 0000000..13fd4d1
--- /dev/null
+++ b/src/com/android/media/tests/VideoMultimeterRunner.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.Option.Importance;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.CommandResult;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Semaphore;
+
+/**
+ * A harness that test video playback with multiple devices and reports result.
+ */
+public class VideoMultimeterRunner extends VideoMultimeterTest
+ implements IDeviceTest, IRemoteTest {
+
+ @Option(name = "robot-util-path", description = "path for robot control util",
+ importance = Importance.ALWAYS, mandatory = true)
+ String mRobotUtilPath = "/tmp/robot_util.sh";
+
+ @Option(name = "device-map", description =
+ "Device serials map to location and audio input. May be repeated",
+ importance = Importance.ALWAYS)
+ Map<String, String> mDeviceMap = new HashMap<String, String>();
+
+ @Option(name = "calibration-map", description =
+ "Device serials map to calibration values. May be repeated",
+ importance = Importance.ALWAYS)
+ Map<String, String> mCalibrationMap = new HashMap<String, String>();
+
+ static final long ROBOT_TIMEOUT_MS = 60 * 1000;
+
+ static final Semaphore runToken = new Semaphore(1);
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ long durationMs = 0;
+ TestIdentifier testId = new TestIdentifier(getClass()
+ .getCanonicalName(), RUN_KEY);
+
+ listener.testRunStarted(RUN_KEY, 0);
+ listener.testStarted(testId);
+
+ long testStartTime = System.currentTimeMillis();
+ Map<String, String> metrics = new HashMap<String, String>();
+
+ try {
+ CLog.v("Waiting to acquire run token");
+ runToken.acquire();
+
+ String deviceSerial = getDevice().getSerialNumber();
+
+ String calibrationValue = (mCalibrationMap.containsKey(deviceSerial) ?
+ mCalibrationMap.get(deviceSerial) : null);
+ if (mDebugWithoutHardware
+ || (moveArm(deviceSerial) && setupTestEnv(calibrationValue))) {
+ runMultimeterTest(listener, metrics);
+ } else {
+ listener.testFailed(testId, "Failed to set up environment");
+ }
+ } catch (InterruptedException e) {
+ CLog.d("Acquire run token interrupted");
+ listener.testFailed(testId, "Failed to acquire run token");
+ } finally {
+ runToken.release();
+ listener.testEnded(testId, metrics);
+ durationMs = System.currentTimeMillis() - testStartTime;
+ listener.testRunEnded(durationMs, metrics);
+ }
+ }
+
+ protected boolean moveArm(String deviceSerial) {
+ if (mDeviceMap.containsKey(deviceSerial)) {
+ CLog.v("Moving robot arm to device " + deviceSerial);
+ CommandResult cr = getRunUtil().runTimedCmd(
+ ROBOT_TIMEOUT_MS, mRobotUtilPath, mDeviceMap.get(deviceSerial));
+ CLog.v(cr.getStdout());
+ return true;
+ } else {
+ CLog.e("Cannot find device in map, test failed");
+ return false;
+ }
+ }
+}
diff --git a/src/com/android/media/tests/VideoMultimeterTest.java b/src/com/android/media/tests/VideoMultimeterTest.java
new file mode 100644
index 0000000..038803f
--- /dev/null
+++ b/src/com/android/media/tests/VideoMultimeterTest.java
@@ -0,0 +1,490 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.media.tests;
+
+import com.android.ddmlib.testrunner.TestIdentifier;
+import com.android.tradefed.config.Option;
+import com.android.tradefed.config.Option.Importance;
+import com.android.tradefed.device.CollectingOutputReceiver;
+import com.android.tradefed.device.DeviceNotAvailableException;
+import com.android.tradefed.device.ITestDevice;
+import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ByteArrayInputStreamSource;
+import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.testtype.IDeviceTest;
+import com.android.tradefed.testtype.IRemoteTest;
+import com.android.tradefed.util.CommandResult;
+import com.android.tradefed.util.IRunUtil;
+import com.android.tradefed.util.RunUtil;
+
+import org.junit.Assert;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A harness that test video playback and reports result.
+ */
+public class VideoMultimeterTest implements IDeviceTest, IRemoteTest {
+
+ static final String RUN_KEY = "video_multimeter";
+
+ @Option(name = "multimeter-util-path", description = "path for multimeter control util",
+ importance = Importance.ALWAYS)
+ String mMeterUtilPath = "/tmp/util.sh";
+
+ @Option(name = "start-video-cmd", description = "adb shell command to start video playback; " +
+ "use '%s' as placeholder for media source filename", importance = Importance.ALWAYS)
+ String mCmdStartVideo = "am instrument -w -r -e media-file"
+ + " \"%s\" -e class com.android.mediaframeworktest.stress.MediaPlayerStressTest"
+ + " com.android.mediaframeworktest/.MediaPlayerStressTestRunner";
+
+ @Option(name = "stop-video-cmd", description = "adb shell command to stop video playback",
+ importance = Importance.ALWAYS)
+ String mCmdStopVideo = "am force-stop com.android.mediaframeworktest";
+
+ @Option(name="video-spec", description=
+ "Comma deliminated information for test video files with the following format: " +
+ "video_filename, reporting_key_prefix, fps, duration(in sec) " +
+ "May be repeated for test with multiple files.")
+ private Collection<String> mVideoSpecs = new ArrayList<>();
+
+ @Option(name="wait-time-between-runs", description=
+ "wait time between two test video measurements, in millisecond")
+ private long mWaitTimeBetweenRuns = 3 * 60 * 1000;
+
+ @Option(name="calibration-video", description=
+ "filename of calibration video")
+ private String mCaliVideoDevicePath = "video_cali.mp4";
+
+ @Option(
+ name = "debug-without-hardware",
+ description = "Use option to debug test without having specialized hardware",
+ importance = Importance.NEVER,
+ mandatory = false
+ )
+ protected boolean mDebugWithoutHardware = false;
+
+ static final String ROTATE_LANDSCAPE = "content insert --uri content://settings/system"
+ + " --bind name:s:user_rotation --bind value:i:1";
+
+ // Max number of trailing frames to trim
+ static final int TRAILING_FRAMES_MAX = 3;
+ // Min threshold for duration of trailing frames
+ static final long FRAME_DURATION_THRESHOLD_US = 500 * 1000; // 0.5s
+
+ static final String CMD_GET_FRAMERATE_STATE = "GETF";
+ static final String CMD_START_CALIBRATION = "STAC";
+ static final String CMD_SET_CALIBRATION_VALS = "SETCAL";
+ static final String CMD_STOP_CALIBRATION = "STOC";
+ static final String CMD_START_MEASUREMENT = "STAM";
+ static final String CMD_STOP_MEASUREMENT = "STOM";
+ static final String CMD_GET_NUM_FRAMES = "GETN";
+ static final String CMD_GET_ALL_DATA = "GETD";
+
+ static final long DEVICE_SYNC_TIME_MS = 30 * 1000;
+ static final long CALIBRATION_TIMEOUT_MS = 30 * 1000;
+ static final long COMMAND_TIMEOUT_MS = 5 * 1000;
+ static final long GETDATA_TIMEOUT_MS = 10 * 60 * 1000;
+
+ // Regex for: "OK (time); (frame duration); (marker color); (total dropped frames)"
+ static final String VIDEO_FRAME_DATA_PATTERN = "OK\\s+\\d+;\\s*(-?\\d+);\\s*[a-z]+;\\s*(\\d+)";
+
+ // Regex for: "OK (time); (frame duration); (marker color); (total dropped frames); (lipsync)"
+ // results in: $1 == ts, $2 == lipsync
+ static final String LIPSYNC_DATA_PATTERN =
+ "OK\\s+(\\d+);\\s*\\d+;\\s*[a-z]+;\\s*\\d+;\\s*(-?\\d+)";
+ // ts dur color missed latency
+ static final int LIPSYNC_SIGNAL = 2000000; // every 2 seconds
+ static final int LIPSYNC_SIGNAL_MIN = 1500000; // must be at least 1.5 seconds after prev
+
+ ITestDevice mDevice;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setDevice(ITestDevice device) {
+ mDevice = device;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ITestDevice getDevice() {
+ return mDevice;
+ }
+
+ private void rotateScreen() throws DeviceNotAvailableException {
+ // rotate to landscape mode, except for manta
+ if (!getDevice().getProductType().contains("manta")) {
+ getDevice().executeShellCommand(ROTATE_LANDSCAPE);
+ }
+ }
+
+ protected boolean setupTestEnv() throws DeviceNotAvailableException {
+ return setupTestEnv(null);
+ }
+
+ protected void startPlayback(final String videoPath) {
+ new Thread() {
+ @Override
+ public void run() {
+ try {
+ CollectingOutputReceiver receiver = new CollectingOutputReceiver();
+ getDevice().executeShellCommand(String.format(
+ mCmdStartVideo, videoPath),
+ receiver, 1L, TimeUnit.SECONDS, 0);
+ } catch (DeviceNotAvailableException e) {
+ CLog.e(e.getMessage());
+ }
+ }
+ }.start();
+ }
+
+ /**
+ * Perform calibration process for video multimeter
+ *
+ * @return boolean whether calibration succeeds
+ * @throws DeviceNotAvailableException
+ */
+ protected boolean doCalibration() throws DeviceNotAvailableException {
+ // play calibration video
+ startPlayback(mCaliVideoDevicePath);
+ getRunUtil().sleep(3 * 1000);
+ rotateScreen();
+ getRunUtil().sleep(1 * 1000);
+ CommandResult cr = getRunUtil().runTimedCmd(
+ COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_START_CALIBRATION);
+ CLog.i("Starting calibration: " + cr.getStdout());
+ // check whether multimeter is calibrated
+ boolean isCalibrated = false;
+ long calibrationStartTime = System.currentTimeMillis();
+ while (!isCalibrated &&
+ System.currentTimeMillis() - calibrationStartTime <= CALIBRATION_TIMEOUT_MS) {
+ getRunUtil().sleep(1 * 1000);
+ cr = getRunUtil().runTimedCmd(2 * 1000, mMeterUtilPath, CMD_GET_FRAMERATE_STATE);
+ if (cr.getStdout().contains("calib0")) {
+ isCalibrated = true;
+ }
+ }
+ if (!isCalibrated) {
+ // stop calibration if time out
+ cr = getRunUtil().runTimedCmd(
+ COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_STOP_CALIBRATION);
+ CLog.e("Calibration timed out.");
+ } else {
+ CLog.i("Calibration succeeds.");
+ }
+ getDevice().executeShellCommand(mCmdStopVideo);
+ return isCalibrated;
+ }
+
+ protected boolean setupTestEnv(String caliValues) throws DeviceNotAvailableException {
+ getRunUtil().sleep(DEVICE_SYNC_TIME_MS);
+ CommandResult cr = getRunUtil().runTimedCmd(
+ COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_STOP_MEASUREMENT);
+
+ getDevice().setDate(new Date());
+ CLog.i("syncing device time to host time");
+ getRunUtil().sleep(3 * 1000);
+
+ // TODO: need a better way to clear old data
+ // start and stop to clear old data
+ cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_START_MEASUREMENT);
+ getRunUtil().sleep(3 * 1000);
+ cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_STOP_MEASUREMENT);
+ getRunUtil().sleep(3 * 1000);
+ CLog.i("Stopping measurement: " + cr.getStdout());
+
+ if (caliValues == null) {
+ return doCalibration();
+ } else {
+ CLog.i("Setting calibration values: " + caliValues);
+ final String calibrationValues = CMD_SET_CALIBRATION_VALS + " " + caliValues;
+ cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, calibrationValues);
+ final String response = mDebugWithoutHardware ? "OK" : cr.getStdout();
+ if (response != null && response.startsWith("OK")) {
+ CLog.i("Calibration values are set to: " + caliValues);
+ return true;
+ } else {
+ CLog.e("Failed to set calibration values: " + cr.getStdout());
+ return false;
+ }
+ }
+ }
+
+ private void doMeasurement(final String testVideoPath, long durationSecond)
+ throws DeviceNotAvailableException {
+ CommandResult cr;
+ getDevice().clearErrorDialogs();
+ getRunUtil().sleep(mWaitTimeBetweenRuns);
+
+ // play test video
+ startPlayback(testVideoPath);
+
+ getRunUtil().sleep(3 * 1000);
+
+ rotateScreen();
+ getRunUtil().sleep(1 * 1000);
+ cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_START_MEASUREMENT);
+ CLog.i("Starting measurement: " + cr.getStdout());
+
+ // end measurement
+ getRunUtil().sleep(durationSecond * 1000);
+
+ cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_STOP_MEASUREMENT);
+ CLog.i("Stopping measurement: " + cr.getStdout());
+ if (cr == null || !cr.getStdout().contains("OK")) {
+ cr = getRunUtil().runTimedCmd(
+ COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_STOP_MEASUREMENT);
+ CLog.i("Retry - Stopping measurement: " + cr.getStdout());
+ }
+
+ getDevice().executeShellCommand(mCmdStopVideo);
+ getDevice().clearErrorDialogs();
+ }
+
+ private Map<String, String> getResult(ITestInvocationListener listener,
+ Map<String, String> metrics, String keyprefix, float fps, boolean lipsync) {
+ CommandResult cr;
+
+ // get number of results
+ getRunUtil().sleep(5 * 1000);
+ cr = getRunUtil().runTimedCmd(COMMAND_TIMEOUT_MS, mMeterUtilPath, CMD_GET_NUM_FRAMES);
+ String frameNum = cr.getStdout();
+
+ CLog.i("== Video Multimeter Result '%s' ==", keyprefix);
+ CLog.i("Number of results: " + frameNum);
+
+ String nrOfDataPointsStr = extractNumberOfCollectedDataPoints(frameNum);
+ metrics.put(keyprefix + "frame_captured", nrOfDataPointsStr);
+
+ long nrOfDataPoints = Long.parseLong(nrOfDataPointsStr);
+
+ Assert.assertTrue("Multimeter did not collect any data for " + keyprefix,
+ nrOfDataPoints > 0);
+
+ CLog.i("Captured frames: " + nrOfDataPointsStr);
+
+ // get all results from multimeter and write to output file
+ cr = getRunUtil().runTimedCmd(GETDATA_TIMEOUT_MS, mMeterUtilPath, CMD_GET_ALL_DATA);
+ String allData = cr.getStdout();
+ listener.testLog(
+ keyprefix, LogDataType.TEXT, new ByteArrayInputStreamSource(allData.getBytes()));
+
+ // parse results
+ return parseResult(metrics, nrOfDataPoints, allData, keyprefix, fps, lipsync);
+ }
+
+ private String extractNumberOfCollectedDataPoints(String numFrames) {
+ // Create pattern that matches string like "OK 14132" capturing the
+ // number of data points.
+ Pattern p = Pattern.compile("OK\\s+(\\d+)$");
+ Matcher m = p.matcher(numFrames.trim());
+
+ String frameCapturedStr = "0";
+ if (m.matches()) {
+ frameCapturedStr = m.group(1);
+ }
+
+ return frameCapturedStr;
+ }
+
+ protected void runMultimeterTest(ITestInvocationListener listener,
+ Map<String,String> metrics) throws DeviceNotAvailableException {
+ for (String videoSpec : mVideoSpecs) {
+ String[] videoInfo = videoSpec.split(",");
+ String filename = videoInfo[0].trim();
+ String keyPrefix = videoInfo[1].trim();
+ float fps = Float.parseFloat(videoInfo[2].trim());
+ long duration = Long.parseLong(videoInfo[3].trim());
+ doMeasurement(filename, duration);
+ metrics = getResult(listener, metrics, keyPrefix, fps, true);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void run(ITestInvocationListener listener)
+ throws DeviceNotAvailableException {
+ TestIdentifier testId = new TestIdentifier(getClass()
+ .getCanonicalName(), RUN_KEY);
+
+ listener.testRunStarted(RUN_KEY, 0);
+ listener.testStarted(testId);
+
+ long testStartTime = System.currentTimeMillis();
+ Map<String, String> metrics = new HashMap<>();
+
+ if (setupTestEnv()) {
+ runMultimeterTest(listener, metrics);
+ }
+
+ long durationMs = System.currentTimeMillis() - testStartTime;
+ listener.testEnded(testId, metrics);
+ listener.testRunEnded(durationMs, metrics);
+ }
+
+ /**
+ * Parse Multimeter result.
+ *
+ * @param result
+ * @return a {@link HashMap} that contains metrics keys and results
+ */
+ private Map<String, String> parseResult(Map<String, String> metrics,
+ long frameCaptured, String result, String keyprefix, float fps,
+ boolean lipsync) {
+ final int MISSING_FRAME_CEILING = 5; //5+ frames missing count the same
+ final double[] MISSING_FRAME_WEIGHT = {0.0, 1.0, 2.5, 5.0, 6.25, 8.0};
+
+ // Get total captured frames and calculate smoothness and freezing score
+ // format: "OK (time); (frame duration); (marker color); (total dropped frames)"
+ Pattern p = Pattern.compile(VIDEO_FRAME_DATA_PATTERN);
+ Matcher m = null;
+ String[] lines = result.split(System.getProperty("line.separator"));
+ String totalDropFrame = "-1";
+ String lastDropFrame = "0";
+ long frameCount = 0;
+ long consecutiveDropFrame = 0;
+ double freezingPenalty = 0.0;
+ long frameDuration = 0;
+ double offByOne = 0;
+ double offByMultiple = 0;
+ double expectedFrameDurationInUs = 1000000.0 / fps;
+ for (int i = 0; i < lines.length; i++) {
+ m = p.matcher(lines[i].trim());
+ if (m.matches()) {
+ frameDuration = Long.parseLong(m.group(1));
+ // frameDuration = -1 indicates dropped frame
+ if (frameDuration > 0) {
+ frameCount++;
+ }
+ totalDropFrame = m.group(2);
+ // trim the last few data points if needed
+ if (frameCount >= frameCaptured - TRAILING_FRAMES_MAX - 1 &&
+ frameDuration > FRAME_DURATION_THRESHOLD_US) {
+ metrics.put(keyprefix + "frame_captured", String.valueOf(frameCount));
+ break;
+ }
+ if (lastDropFrame.equals(totalDropFrame)) {
+ if (consecutiveDropFrame > 0) {
+ freezingPenalty += MISSING_FRAME_WEIGHT[(int) (Math.min(consecutiveDropFrame,
+ MISSING_FRAME_CEILING))] * consecutiveDropFrame;
+ consecutiveDropFrame = 0;
+ }
+ } else {
+ consecutiveDropFrame++;
+ }
+ lastDropFrame = totalDropFrame;
+
+ if (frameDuration < expectedFrameDurationInUs * 0.5) {
+ offByOne++;
+ } else if (frameDuration > expectedFrameDurationInUs * 1.5) {
+ if (frameDuration < expectedFrameDurationInUs * 2.5) {
+ offByOne++;
+ } else {
+ offByMultiple++;
+ }
+ }
+ }
+ }
+ if (totalDropFrame.equals("-1")) {
+ // no matching result found
+ CLog.w("No result found for " + keyprefix);
+ return metrics;
+ } else {
+ metrics.put(keyprefix + "frame_drop", totalDropFrame);
+ CLog.i("Dropped frames: " + totalDropFrame);
+ }
+ double smoothnessScore = 100.0 - (offByOne / frameCaptured) * 100.0 -
+ (offByMultiple / frameCaptured) * 300.0;
+ metrics.put(keyprefix + "smoothness", String.valueOf(smoothnessScore));
+ CLog.i("Off by one frame: " + offByOne);
+ CLog.i("Off by multiple frames: " + offByMultiple);
+ CLog.i("Smoothness score: " + smoothnessScore);
+
+ double freezingScore = 100.0 - 100.0 * freezingPenalty / frameCaptured;
+ metrics.put(keyprefix + "freezing", String.valueOf(freezingScore));
+ CLog.i("Freezing score: " + freezingScore);
+
+ // parse lipsync results (the audio and video synchronization offset)
+ // format: "OK (time); (frame duration); (marker color); (total dropped frames); (lipsync)"
+ if (lipsync) {
+ ArrayList<Integer> lipsyncVals = new ArrayList<>();
+ StringBuilder lipsyncValsStr = new StringBuilder("[");
+ long lipsyncSum = 0;
+ int lipSyncLastTime = -1;
+
+ Pattern pLip = Pattern.compile(LIPSYNC_DATA_PATTERN);
+ for (int i = 0; i < lines.length; i++) {
+ m = pLip.matcher(lines[i].trim());
+ if (m.matches()) {
+ int lipSyncTime = Integer.parseInt(m.group(1));
+ int lipSyncVal = Integer.parseInt(m.group(2));
+ if (lipSyncLastTime != -1) {
+ if ((lipSyncTime - lipSyncLastTime) < LIPSYNC_SIGNAL_MIN) {
+ continue; // ignore the early/spurious one
+ }
+ }
+ lipSyncLastTime = lipSyncTime;
+
+ lipsyncVals.add(lipSyncVal);
+ lipsyncValsStr.append(lipSyncVal);
+ lipsyncValsStr.append(", ");
+ lipsyncSum += lipSyncVal;
+ }
+ }
+ if (lipsyncVals.size() > 0) {
+ lipsyncValsStr.append("]");
+ CLog.i("Lipsync values: " + lipsyncValsStr);
+ Collections.sort(lipsyncVals);
+ int lipsyncCount = lipsyncVals.size();
+ int minLipsync = lipsyncVals.get(0);
+ int maxLipsync = lipsyncVals.get(lipsyncCount - 1);
+ metrics.put(keyprefix + "lipsync_count", String.valueOf(lipsyncCount));
+ CLog.i("Lipsync Count: " + lipsyncCount);
+ metrics.put(keyprefix + "lipsync_min", String.valueOf(lipsyncVals.get(0)));
+ CLog.i("Lipsync Min: " + minLipsync);
+ metrics.put(keyprefix + "lipsync_max", String.valueOf(maxLipsync));
+ CLog.i("Lipsync Max: " + maxLipsync);
+ double meanLipSync = (double) lipsyncSum / lipsyncCount;
+ metrics.put(keyprefix + "lipsync_mean", String.valueOf(meanLipSync));
+ CLog.i("Lipsync Mean: " + meanLipSync);
+ } else {
+ CLog.w("Lipsync value not found in result.");
+ }
+ }
+ CLog.i("== End ==", keyprefix);
+ return metrics;
+ }
+
+ protected IRunUtil getRunUtil() {
+ return RunUtil.getDefault();
+ }
+}