| /* |
| * Copyright (C) 2011 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.android.media.tests; |
| |
| import com.android.ddmlib.IDevice; |
| import com.android.ddmlib.Log; |
| import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; |
| import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; |
| import com.android.tradefed.device.DeviceNotAvailableException; |
| import com.android.tradefed.device.ITestDevice; |
| import com.android.tradefed.result.CollectingTestListener; |
| import com.android.tradefed.result.ITestInvocationListener; |
| import com.android.tradefed.result.InputStreamSource; |
| import com.android.tradefed.result.LogDataType; |
| import com.android.tradefed.result.SnapshotInputStreamSource; |
| import com.android.tradefed.testtype.IDeviceTest; |
| import com.android.tradefed.testtype.IRemoteTest; |
| import com.android.tradefed.util.RegexTrie; |
| import com.android.tradefed.util.StreamUtil; |
| |
| import junit.framework.Assert; |
| import junit.framework.TestCase; |
| |
| import java.io.ByteArrayInputStream; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.HashMap; |
| import java.util.List; |
| import java.util.ListIterator; |
| import java.util.Map; |
| import java.util.Set; |
| |
| /** |
| * Runs the Camera stress testcases. |
| * FIXME: more details |
| * <p/> |
| * Note that this test will not run properly unless /sdcard is mounted and writable. |
| */ |
| public class CameraStressTest implements IDeviceTest, IRemoteTest { |
| private static final String LOG_TAG = "CameraStressTest"; |
| |
| ITestDevice mTestDevice = null; |
| |
| // Constants for running the tests |
| private static final String TEST_PACKAGE_NAME = "com.google.android.gallery3d.tests"; |
| |
| //Max test timeout - 3 hrs |
| private static final int MAX_TEST_TIMEOUT = 3 * 60 * 60 * 1000; |
| |
| private final String mOutputPath = "mediaStressOut.txt"; |
| |
| /** |
| * Stores the test cases that we should consider running. |
| * <p/> |
| * This currently consists of "startup" and "latency" |
| */ |
| private List<TestInfo> mTestCases = new ArrayList<TestInfo>(); |
| |
| /** |
| * A struct that contains useful info about the tests to run |
| */ |
| static class TestInfo { |
| public String mTestName = null; |
| public String mClassName = null; |
| public String mTestMetricsName = null; |
| public Map<String, String> mInstrumentationArgs = new HashMap<String, String>(); |
| public RegexTrie<String> mPatternMap = new RegexTrie<String>(); |
| |
| @Override |
| public String toString() { |
| return String.format("TestInfo: name(%s) class(%s) metric(%s) patterns(%s)", mTestName, |
| mClassName, mTestMetricsName, mPatternMap); |
| } |
| } |
| |
| /** |
| * Set up the pattern map for parsing output files |
| * <p/> |
| * Exposed for unit meta-testing |
| */ |
| static RegexTrie<String> getPatternMap() { |
| RegexTrie<String> patMap = new RegexTrie<String>(); |
| patMap.put("SwitchPreview", "^Camera Switch Mode:"); |
| |
| // For versions of the on-device test that don't differentiate between front and back camera |
| patMap.put("ImageCapture", "^Camera Image Capture"); |
| patMap.put("VideoRecording", "^Camera Video Capture"); |
| |
| // For versions that do differentiate |
| patMap.put("FrontImageCapture", "^Front Camera Image Capture"); |
| patMap.put("ImageCapture", "^Back Camera Image Capture"); |
| patMap.put("FrontVideoRecording", "^Front Camera Video Capture"); |
| patMap.put("VideoRecording", "^Back Camera Video Capture"); |
| |
| // Actual metrics to collect for a given key |
| patMap.put("loopCount", "^No of loops :(\\d+)"); |
| patMap.put("iters", "^loop:.+,(\\d+)"); |
| |
| return patMap; |
| } |
| |
| /** |
| * Set up the configurations for the test cases we want to run |
| */ |
| public CameraStressTest() { |
| RegexTrie<String> patMap = getPatternMap(); |
| |
| // Image capture stress test |
| TestInfo t = new TestInfo(); |
| t.mTestName = "imagecap"; |
| t.mClassName = "com.android.gallery3d.stress.ImageCapture"; |
| t.mTestMetricsName = "CameraApplicationStress"; |
| t.mInstrumentationArgs.put("image_iterations", Integer.toString(100)); |
| t.mPatternMap = patMap; |
| mTestCases.add(t); |
| |
| // Image capture stress test |
| t = new TestInfo(); |
| t.mTestName = "videocap"; |
| t.mClassName = "com.android.gallery3d.stress.VideoCapture"; |
| t.mTestMetricsName = "CameraApplicationStress"; |
| t.mInstrumentationArgs.put("video_iterations", Integer.toString(100)); |
| t.mPatternMap = patMap; |
| mTestCases.add(t); |
| |
| // "SwitchPreview" stress test |
| t = new TestInfo(); |
| t.mTestName = "switch"; |
| t.mClassName = "com.android.gallery3d.stress.SwitchPreview"; |
| t.mTestMetricsName = "CameraApplicationStress"; |
| t.mPatternMap = patMap; |
| mTestCases.add(t); |
| } |
| |
| @Override |
| public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { |
| Assert.assertNotNull(mTestDevice); |
| |
| for (TestInfo test : mTestCases) { |
| cleanTmpFiles(); |
| executeTest(test, listener); |
| logOutputFiles(test, listener); |
| } |
| |
| cleanTmpFiles(); |
| } |
| |
| private void executeTest(TestInfo test, ITestInvocationListener listener) |
| throws DeviceNotAvailableException { |
| IRemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(TEST_PACKAGE_NAME, |
| mTestDevice.getIDevice()); |
| CollectingTestListener auxListener = new CollectingTestListener(); |
| |
| runner.setClassName(test.mClassName); |
| runner.setMaxtimeToOutputResponse(MAX_TEST_TIMEOUT); |
| |
| Set<String> argumentKeys = test.mInstrumentationArgs.keySet(); |
| for (String s : argumentKeys) { |
| runner.addInstrumentationArg(s, test.mInstrumentationArgs.get(s)); |
| } |
| |
| mTestDevice.runInstrumentationTests(runner, listener, auxListener); |
| |
| // Grab a bugreport if warranted |
| if (auxListener.hasFailedTests()) { |
| Log.e(LOG_TAG, String.format("Grabbing bugreport after test '%s' finished with " + |
| "%d failures and %d errors.", test.mTestName, auxListener.getNumFailedTests(), |
| auxListener.getNumErrorTests())); |
| InputStreamSource bugreport = mTestDevice.getBugreport(); |
| listener.testLog(String.format("bugreport-%s.txt", test.mTestName), LogDataType.TEXT, |
| bugreport); |
| bugreport.cancel(); |
| } |
| } |
| |
| /** |
| * Clean up temp files from test runs |
| * <p /> |
| * Note that all photos on the test device will be removed |
| */ |
| private void cleanTmpFiles() throws DeviceNotAvailableException { |
| String extStore = mTestDevice.getMountPoint(IDevice.MNT_EXTERNAL_STORAGE); |
| mTestDevice.executeShellCommand(String.format("rm -r %s/DCIM", extStore)); |
| mTestDevice.executeShellCommand(String.format("rm %s/%s", extStore, mOutputPath)); |
| } |
| |
| /** |
| * Pull the output file from the device, add it to the logs, and also parse out the relevant |
| * test metrics and report them. Additionally, pull the memory file (if it exists) and report |
| * it. |
| */ |
| private void logOutputFiles(TestInfo test, ITestInvocationListener listener) |
| throws DeviceNotAvailableException { |
| File outputFile = null; |
| InputStreamSource outputSource = null; |
| try { |
| outputFile = mTestDevice.pullFileFromExternal(mOutputPath); |
| |
| if (outputFile == null) { |
| return; |
| } |
| |
| // Upload a verbatim copy of the output file |
| Log.d(LOG_TAG, String.format("Sending %d byte file %s into the logosphere!", |
| outputFile.length(), outputFile)); |
| outputSource = new SnapshotInputStreamSource(new FileInputStream(outputFile)); |
| listener.testLog(String.format("output-%s.txt", test.mTestName), LogDataType.TEXT, |
| outputSource); |
| |
| // Parse the output file to upload aggregated metrics |
| parseOutputFile(test, new FileInputStream(outputFile), listener); |
| } catch (IOException e) { |
| Log.e(LOG_TAG, String.format("IOException while reading or parsing output file: %s", e)); |
| } finally { |
| if (outputFile != null) { |
| outputFile.delete(); |
| } |
| if (outputSource != null) { |
| outputSource.cancel(); |
| } |
| } |
| } |
| |
| /** |
| * Parse the relevant metrics from the Instrumentation test output file |
| */ |
| private void parseOutputFile(TestInfo test, InputStream dataStream, |
| ITestInvocationListener listener) { |
| Map<String, String> runMetrics = new HashMap<String, String>(); |
| |
| String contents; |
| try { |
| contents = StreamUtil.getStringFromStream(dataStream); |
| } catch (IOException e) { |
| Log.e(LOG_TAG, String.format("Got IOException during %s test processing: %s", |
| test.mTestName, e)); |
| return; |
| } |
| |
| String key = null; |
| Integer countExpected = null; |
| Integer countActual = null; |
| |
| List<String> lines = Arrays.asList(contents.split("\n")); |
| ListIterator<String> lineIter = lines.listIterator(); |
| String line; |
| while (lineIter.hasNext()) { |
| line = lineIter.next(); |
| List<List<String>> capture = new ArrayList<List<String>>(1); |
| String pattern = test.mPatternMap.retrieve(capture, line); |
| if (pattern != null) { |
| if ("loopCount".equals(pattern)) { |
| // First capture in first (only) string |
| countExpected = Integer.parseInt(capture.get(0).get(0)); |
| } else if ("iters".equals(pattern)) { |
| // First capture in first (only) string |
| countActual = Integer.parseInt(capture.get(0).get(0)); |
| |
| if (countActual != null) { |
| // countActual starts counting at 0 |
| countActual += 1; |
| } |
| } else { |
| // Assume that the pattern is the name of a key |
| |
| // commit, if there was a previous key |
| if (key != null) { |
| int value = coalesceLoopCounts(countActual, countExpected); |
| runMetrics.put(key, Integer.toString(value)); |
| } |
| |
| key = pattern; |
| countExpected = null; |
| countActual = null; |
| } |
| |
| Log.d(LOG_TAG, String.format("Got %s key '%s' and captures '%s'", |
| test.mTestName, key, capture.toString())); |
| } else if (line.isEmpty()) { |
| // ignore |
| continue; |
| } else { |
| Log.e(LOG_TAG, String.format("Got unmatched line: %s", line)); |
| continue; |
| } |
| |
| // commit the final key, if there was one |
| if (key != null) { |
| int value = coalesceLoopCounts(countActual, countExpected); |
| runMetrics.put(key, Integer.toString(value)); |
| } |
| } |
| |
| reportMetrics(listener, test, runMetrics); |
| } |
| |
| /** |
| * Given an actual and an expected iteration count, determine a single metric to report. |
| */ |
| private int coalesceLoopCounts(Integer actual, Integer expected) { |
| if (expected == null || expected <= 0) { |
| return -1; |
| } else if (actual == null) { |
| return expected; |
| } else { |
| return actual; |
| } |
| } |
| |
| /** |
| * Report run metrics by creating an empty test run to stick them in |
| * <p /> |
| * Exposed for unit testing |
| */ |
| void reportMetrics(ITestInvocationListener listener, TestInfo test, |
| Map<String, String> metrics) { |
| // Create an empty testRun to report the parsed runMetrics |
| Log.e(LOG_TAG, String.format("About to report metrics for %s: %s", test.mTestMetricsName, |
| metrics)); |
| listener.testRunStarted(test.mTestMetricsName, 0); |
| listener.testRunEnded(0, metrics); |
| } |
| |
| @Override |
| public void setDevice(ITestDevice device) { |
| mTestDevice = device; |
| } |
| |
| @Override |
| public ITestDevice getDevice() { |
| return mTestDevice; |
| } |
| |
| /** |
| * A meta-test to ensure that bits of the BluetoothStressTest are working properly |
| */ |
| public static class MetaTest extends TestCase { |
| private CameraStressTest mTestInstance = null; |
| |
| private TestInfo mTestInfo = null; |
| |
| private TestInfo mReportedTestInfo = null; |
| private Map<String, String> mReportedMetrics = null; |
| |
| private static String join(String... pieces) { |
| StringBuilder sb = new StringBuilder(); |
| for (String piece : pieces) { |
| sb.append(piece); |
| sb.append("\n"); |
| } |
| return sb.toString(); |
| } |
| |
| @Override |
| public void setUp() throws Exception { |
| mTestInstance = new CameraStressTest() { |
| @Override |
| void reportMetrics(ITestInvocationListener l, TestInfo test, |
| Map<String, String> metrics) { |
| mReportedTestInfo = test; |
| mReportedMetrics = metrics; |
| } |
| }; |
| |
| // Image capture stress test |
| mTestInfo = new TestInfo(); |
| TestInfo t = mTestInfo; // for convenience |
| t.mTestName = "capture"; |
| t.mClassName = "com.android.gallery3d.stress.ImageCapture"; |
| t.mTestMetricsName = "camera_application_stress"; |
| t.mPatternMap = getPatternMap(); |
| } |
| |
| /** |
| * Make sure that parsing works for devices sending output in the old format |
| */ |
| public void testParse_old() throws Exception { |
| String output = join( |
| "Camera Image Capture", |
| "No of loops :100", |
| "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + |
| ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 " + |
| ",36 ,37 ,38 ,39 ,40 ,41 ,42", |
| "Camera Video Capture", |
| "No of loops :100", |
| "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + |
| ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 " + |
| ",36 ,37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 " + |
| ",53 ,54 ,55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 " + |
| ",70 ,71 ,72 ,73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 " + |
| ",87 ,88 ,89 ,90 ,91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99", |
| "Camera Switch Mode:", |
| "No of loops :200", |
| "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13"); |
| |
| InputStream iStream = new ByteArrayInputStream(output.getBytes()); |
| mTestInstance.parseOutputFile(mTestInfo, iStream, null); |
| assertEquals(mTestInfo, mReportedTestInfo); |
| assertNotNull(mReportedMetrics); |
| Log.e(LOG_TAG, String.format("Got reported metrics: %s", mReportedMetrics.toString())); |
| assertEquals(3, mReportedMetrics.size()); |
| assertEquals("43", mReportedMetrics.get("ImageCapture")); |
| assertEquals("100", mReportedMetrics.get("VideoRecording")); |
| assertEquals("14", mReportedMetrics.get("SwitchPreview")); |
| } |
| |
| /** |
| * Make sure that parsing works for devices sending output in the new format |
| */ |
| public void testParse_new() throws Exception { |
| String output = join( |
| "Camera Stress Test result", |
| "/auto/android-test/data/CameraStressTest_git_honeycomb-mr1-release_" + |
| "1700614441c02617_109535_CameraStressOut.txt", |
| "Back Camera Image Capture", |
| "No of loops :100", |
| "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + |
| ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " + |
| ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " + |
| ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " + |
| ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " + |
| ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99", |
| "Front Camera Image Capture", |
| "No of loops :100", |
| "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + |
| ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " + |
| ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " + |
| ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " + |
| ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " + |
| ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98", |
| "Back Camera Video Capture", |
| "No of loops :100", |
| "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + |
| ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " + |
| ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " + |
| ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " + |
| ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " + |
| ",91 ,92 ,93 ,94 ,95 ,96 ,97", |
| "Front Camera Video Capture", |
| "No of loops :100", |
| "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + |
| ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " + |
| ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " + |
| ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " + |
| ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " + |
| ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99", |
| "Camera Switch Mode:", |
| "No of loops :200", |
| "loop: ,0 ,1 ,2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 ,12 ,13 ,14 ,15 ,16 ,17 ,18 " + |
| ",19 ,20 ,21 ,22 ,23 ,24 ,25 ,26 ,27 ,28 ,29 ,30 ,31 ,32 ,33 ,34 ,35 ,36 " + |
| ",37 ,38 ,39 ,40 ,41 ,42 ,43 ,44 ,45 ,46 ,47 ,48 ,49 ,50 ,51 ,52 ,53 ,54 " + |
| ",55 ,56 ,57 ,58 ,59 ,60 ,61 ,62 ,63 ,64 ,65 ,66 ,67 ,68 ,69 ,70 ,71 ,72 " + |
| ",73 ,74 ,75 ,76 ,77 ,78 ,79 ,80 ,81 ,82 ,83 ,84 ,85 ,86 ,87 ,88 ,89 ,90 " + |
| ",91 ,92 ,93 ,94 ,95 ,96 ,97 ,98 ,99 ,100 ,101 ,102 ,103 ,104 ,105 ,106 " + |
| ",107 ,108 ,109 ,110 ,111 ,112 ,113 ,114 ,115 ,116 ,117 ,118 ,119 ,120 " + |
| ",121 ,122 ,123 ,124 ,125 ,126 ,127 ,128 ,129 ,130 ,131 ,132 ,133 ,134 " + |
| ",135 ,136 ,137 ,138 ,139 ,140 ,141 ,142 ,143 ,144 ,145 ,146 ,147 ,148 " + |
| ",149 ,150 ,151 ,152 ,153 ,154 ,155 ,156 ,157 ,158 ,159 ,160 ,161 ,162 " + |
| ",163 ,164 ,165 ,166 ,167 ,168 ,169 ,170 ,171 ,172 ,173 ,174 ,175 ,176 " + |
| ",177 ,178 ,179 ,180 ,181 ,182 ,183 ,184 ,185 ,186 ,187 ,188 ,189 ,190 " + |
| ",191 ,192 ,193 ,194 ,195 ,196 ,197 ,198 ,199"); |
| |
| InputStream iStream = new ByteArrayInputStream(output.getBytes()); |
| mTestInstance.parseOutputFile(mTestInfo, iStream, null); |
| assertEquals(mTestInfo, mReportedTestInfo); |
| assertNotNull(mReportedMetrics); |
| Log.e(LOG_TAG, String.format("Got reported metrics: %s", mReportedMetrics.toString())); |
| assertEquals(5, mReportedMetrics.size()); |
| assertEquals("100", mReportedMetrics.get("ImageCapture")); |
| assertEquals("99", mReportedMetrics.get("FrontImageCapture")); |
| assertEquals("98", mReportedMetrics.get("VideoRecording")); |
| assertEquals("100", mReportedMetrics.get("FrontVideoRecording")); |
| assertEquals("200", mReportedMetrics.get("SwitchPreview")); |
| } |
| } |
| } |
| |