Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2018 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package android.perftests.utils; |
| 18 | |
| 19 | import android.app.Activity; |
| 20 | import android.app.Instrumentation; |
| 21 | import android.os.Bundle; |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 22 | import android.util.ArrayMap; |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 23 | import android.util.Log; |
| 24 | |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 25 | import java.lang.annotation.ElementType; |
| 26 | import java.lang.annotation.Retention; |
| 27 | import java.lang.annotation.RetentionPolicy; |
| 28 | import java.lang.annotation.Target; |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 29 | import java.util.ArrayList; |
| 30 | import java.util.concurrent.TimeUnit; |
| 31 | |
| 32 | /** |
| 33 | * Provides a benchmark framework. |
| 34 | * |
| 35 | * This differs from BenchmarkState in that rather than the class measuring the the elapsed time, |
| 36 | * the test passes in the elapsed time. |
| 37 | * |
| 38 | * Example usage: |
| 39 | * |
| 40 | * public void sampleMethod() { |
| 41 | * ManualBenchmarkState state = new ManualBenchmarkState(); |
| 42 | * |
| 43 | * int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; |
| 44 | * long elapsedTime = 0; |
| 45 | * while (state.keepRunning(elapsedTime)) { |
| 46 | * long startTime = System.nanoTime(); |
| 47 | * int[] dest = new int[src.length]; |
| 48 | * System.arraycopy(src, 0, dest, 0, src.length); |
| 49 | * elapsedTime = System.nanoTime() - startTime; |
| 50 | * } |
| 51 | * System.out.println(state.summaryLine()); |
| 52 | * } |
| 53 | * |
| 54 | * Or use the PerfManualStatusReporter TestRule. |
| 55 | * |
| 56 | * Make sure that the overhead of checking the clock does not noticeably affect the results. |
| 57 | */ |
| 58 | public final class ManualBenchmarkState { |
| 59 | private static final String TAG = ManualBenchmarkState.class.getSimpleName(); |
| 60 | |
| 61 | // TODO: Tune these values. |
| 62 | // warm-up for duration |
| 63 | private static final long WARMUP_DURATION_NS = TimeUnit.SECONDS.toNanos(5); |
| 64 | // minimum iterations to warm-up for |
| 65 | private static final int WARMUP_MIN_ITERATIONS = 8; |
| 66 | |
| 67 | // target testing for duration |
| 68 | private static final long TARGET_TEST_DURATION_NS = TimeUnit.SECONDS.toNanos(16); |
| 69 | private static final int MAX_TEST_ITERATIONS = 1000000; |
| 70 | private static final int MIN_TEST_ITERATIONS = 10; |
| 71 | |
| 72 | private static final int NOT_STARTED = 0; // The benchmark has not started yet. |
| 73 | private static final int WARMUP = 1; // The benchmark is warming up. |
| 74 | private static final int RUNNING = 2; // The benchmark is running. |
| 75 | private static final int FINISHED = 3; // The benchmark has stopped. |
| 76 | |
| 77 | private int mState = NOT_STARTED; // Current benchmark state. |
| 78 | |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 79 | private long mWarmupDurationNs = WARMUP_DURATION_NS; |
| 80 | private long mTargetTestDurationNs = TARGET_TEST_DURATION_NS; |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 81 | private long mWarmupStartTime = 0; |
| 82 | private int mWarmupIterations = 0; |
| 83 | |
| 84 | private int mMaxIterations = 0; |
| 85 | |
| 86 | // Individual duration in nano seconds. |
| 87 | private ArrayList<Long> mResults = new ArrayList<>(); |
| 88 | |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 89 | /** @see #addExtraResult(String, long) */ |
| 90 | private ArrayMap<String, ArrayList<Long>> mExtraResults; |
| 91 | |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 92 | // Statistics. These values will be filled when the benchmark has finished. |
| 93 | // The computation needs double precision, but long int is fine for final reporting. |
| 94 | private Stats mStats; |
| 95 | |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 96 | void configure(ManualBenchmarkTest testAnnotation) { |
| 97 | if (testAnnotation == null) { |
| 98 | return; |
| 99 | } |
| 100 | |
| 101 | final long warmupDurationNs = testAnnotation.warmupDurationNs(); |
| 102 | if (warmupDurationNs >= 0) { |
| 103 | mWarmupDurationNs = warmupDurationNs; |
| 104 | } |
| 105 | final long targetTestDurationNs = testAnnotation.targetTestDurationNs(); |
| 106 | if (targetTestDurationNs >= 0) { |
| 107 | mTargetTestDurationNs = targetTestDurationNs; |
| 108 | } |
| 109 | } |
| 110 | |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 111 | private void beginBenchmark(long warmupDuration, int iterations) { |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 112 | mMaxIterations = (int) (mTargetTestDurationNs / (warmupDuration / iterations)); |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 113 | mMaxIterations = Math.min(MAX_TEST_ITERATIONS, |
| 114 | Math.max(mMaxIterations, MIN_TEST_ITERATIONS)); |
| 115 | mState = RUNNING; |
| 116 | } |
| 117 | |
| 118 | /** |
| 119 | * Judges whether the benchmark needs more samples. |
| 120 | * |
| 121 | * For the usage, see class comment. |
| 122 | */ |
| 123 | public boolean keepRunning(long duration) { |
| 124 | if (duration < 0) { |
| 125 | throw new RuntimeException("duration is negative: " + duration); |
| 126 | } |
| 127 | switch (mState) { |
| 128 | case NOT_STARTED: |
| 129 | mState = WARMUP; |
| 130 | mWarmupStartTime = System.nanoTime(); |
| 131 | return true; |
| 132 | case WARMUP: { |
| 133 | final long timeSinceStartingWarmup = System.nanoTime() - mWarmupStartTime; |
| 134 | ++mWarmupIterations; |
| 135 | if (mWarmupIterations >= WARMUP_MIN_ITERATIONS |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 136 | && timeSinceStartingWarmup >= mWarmupDurationNs) { |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 137 | beginBenchmark(timeSinceStartingWarmup, mWarmupIterations); |
| 138 | } |
| 139 | return true; |
| 140 | } |
| 141 | case RUNNING: { |
| 142 | mResults.add(duration); |
| 143 | final boolean keepRunning = mResults.size() < mMaxIterations; |
| 144 | if (!keepRunning) { |
| 145 | mStats = new Stats(mResults); |
| 146 | mState = FINISHED; |
| 147 | } |
| 148 | return keepRunning; |
| 149 | } |
| 150 | case FINISHED: |
| 151 | throw new IllegalStateException("The benchmark has finished."); |
| 152 | default: |
| 153 | throw new IllegalStateException("The benchmark is in an unknown state."); |
| 154 | } |
| 155 | } |
| 156 | |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 157 | /** |
| 158 | * Adds additional result while this benchmark is running. It is used when a sequence of |
| 159 | * operations is executed consecutively, the duration of each operation can also be recorded. |
| 160 | */ |
| 161 | public void addExtraResult(String key, long duration) { |
| 162 | if (mState != RUNNING) { |
| 163 | return; |
| 164 | } |
| 165 | if (mExtraResults == null) { |
| 166 | mExtraResults = new ArrayMap<>(); |
| 167 | } |
| 168 | mExtraResults.computeIfAbsent(key, k -> new ArrayList<>()).add(duration); |
| 169 | } |
| 170 | |
| 171 | private static String summaryLine(String key, Stats stats, ArrayList<Long> results) { |
| 172 | final StringBuilder sb = new StringBuilder(key); |
| 173 | sb.append(" Summary: "); |
| 174 | sb.append("median=").append(stats.getMedian()).append("ns, "); |
| 175 | sb.append("mean=").append(stats.getMean()).append("ns, "); |
| 176 | sb.append("min=").append(stats.getMin()).append("ns, "); |
| 177 | sb.append("max=").append(stats.getMax()).append("ns, "); |
| 178 | sb.append("sigma=").append(stats.getStandardDeviation()).append(", "); |
| 179 | sb.append("iteration=").append(results.size()).append(", "); |
| 180 | sb.append("values="); |
| 181 | if (results.size() > 100) { |
| 182 | sb.append(results.subList(0, 100)).append(" ..."); |
| 183 | } else { |
| 184 | sb.append(results); |
| 185 | } |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 186 | return sb.toString(); |
| 187 | } |
| 188 | |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 189 | private static void fillStatus(Bundle status, String key, Stats stats) { |
| 190 | status.putLong(key + "_median", stats.getMedian()); |
| 191 | status.putLong(key + "_mean", (long) stats.getMean()); |
| 192 | status.putLong(key + "_percentile90", stats.getPercentile90()); |
| 193 | status.putLong(key + "_percentile95", stats.getPercentile95()); |
| 194 | status.putLong(key + "_stddev", (long) stats.getStandardDeviation()); |
| 195 | } |
| 196 | |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 197 | public void sendFullStatusReport(Instrumentation instrumentation, String key) { |
| 198 | if (mState != FINISHED) { |
| 199 | throw new IllegalStateException("The benchmark hasn't finished"); |
| 200 | } |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 201 | Log.i(TAG, summaryLine(key, mStats, mResults)); |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 202 | final Bundle status = new Bundle(); |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 203 | fillStatus(status, key, mStats); |
| 204 | if (mExtraResults != null) { |
| 205 | for (int i = 0; i < mExtraResults.size(); i++) { |
| 206 | final String subKey = key + "_" + mExtraResults.keyAt(i); |
| 207 | final Stats stats = new Stats(mExtraResults.valueAt(i)); |
| 208 | Log.i(TAG, summaryLine(subKey, mStats, mResults)); |
| 209 | fillStatus(status, subKey, stats); |
| 210 | } |
| 211 | } |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 212 | instrumentation.sendStatus(Activity.RESULT_OK, status); |
| 213 | } |
Arthur Eubanks | 263d674 | 2017-12-18 13:46:59 -0800 | [diff] [blame] | 214 | |
Riddle Hsu | 54a86c6 | 2019-07-10 21:37:08 +0800 | [diff] [blame^] | 215 | /** The annotation to customize the test, e.g. the duration of warm-up and target test. */ |
| 216 | @Target(ElementType.METHOD) |
| 217 | @Retention(RetentionPolicy.RUNTIME) |
| 218 | public @interface ManualBenchmarkTest { |
| 219 | long warmupDurationNs() default -1; |
| 220 | long targetTestDurationNs() default -1; |
| 221 | } |
| 222 | } |