Greatly reduce BenchmarkState overhead
CL reduces BenchmarkState to minimal levels. It also adds
a warmup loop to get things going first before starting measurements.
With this change with clocks /not/ locked on bullhead the test for
RenderNodeJniOverhead is showing a stable (0ns std dev) result
of 54ns, which is approximately the expected amount.
Test: Ran a few perf benchmarks
Change-Id: If01e455884711ebd9cfb89f076efa19dc0b5436d
diff --git a/apct-tests/perftests/core/src/java/lang/perftests/SystemPerfTest.java b/apct-tests/perftests/core/src/java/lang/perftests/SystemPerfTest.java
index 6a49c03..80ce22e 100644
--- a/apct-tests/perftests/core/src/java/lang/perftests/SystemPerfTest.java
+++ b/apct-tests/perftests/core/src/java/lang/perftests/SystemPerfTest.java
@@ -37,4 +37,10 @@
System.nanoTime();
}
}
+
+ @Test
+ public void testBenchmarkOverhead() {
+ BenchmarkState state = mPerfStatusReporter.getBenchmarkState();
+ while (state.keepRunning()) {}
+ }
}
diff --git a/apct-tests/perftests/multiuser/src/android/multiuser/UserLifecycleTest.java b/apct-tests/perftests/multiuser/src/android/multiuser/UserLifecycleTest.java
index cdbca63..88cb8e6 100644
--- a/apct-tests/perftests/multiuser/src/android/multiuser/UserLifecycleTest.java
+++ b/apct-tests/perftests/multiuser/src/android/multiuser/UserLifecycleTest.java
@@ -57,8 +57,6 @@
@LargeTest
@RunWith(AndroidJUnit4.class)
public class UserLifecycleTest {
- private final int MIN_REPEAT_TIMES = 4;
-
private final int TIMEOUT_REMOVE_USER_MS = 4 * 1000; // 4 sec
private final int CHECK_USER_REMOVED_INTERVAL_MS = 200; // 0.2 sec
@@ -90,7 +88,6 @@
mAm = context.getSystemService(ActivityManager.class);
mIam = ActivityManagerNative.getDefault();
mState = mPerfStatusReporter.getBenchmarkState();
- mState.setMinRepeatTimes(MIN_REPEAT_TIMES);
mUsersToRemove = new ArrayList<>();
}
@@ -300,4 +297,4 @@
mUsersToRemove.add(userId);
}
}
-}
\ No newline at end of file
+}
diff --git a/apct-tests/perftests/utils/src/android/perftests/utils/BenchmarkState.java b/apct-tests/perftests/utils/src/android/perftests/utils/BenchmarkState.java
index b27d71b..bf04f6d 100644
--- a/apct-tests/perftests/utils/src/android/perftests/utils/BenchmarkState.java
+++ b/apct-tests/perftests/utils/src/android/perftests/utils/BenchmarkState.java
@@ -23,6 +23,7 @@
import java.util.ArrayList;
import java.util.Collections;
+import java.util.concurrent.TimeUnit;
/**
* Provides a benchmark framework.
@@ -41,39 +42,47 @@
* System.out.println(state.summaryLine());
* }
*/
-public class BenchmarkState {
+public final class BenchmarkState {
private static final String TAG = "BenchmarkState";
- private static final int NOT_STARTED = 1; // The benchmark has not started yet.
+ private static final int NOT_STARTED = 0; // The benchmark has not started yet.
+ private static final int WARMUP = 1; // The benchmark is warming up.
private static final int RUNNING = 2; // The benchmark is running.
private static final int RUNNING_PAUSED = 3; // The benchmark is temporary paused.
private static final int FINISHED = 4; // The benchmark has stopped.
private int mState = NOT_STARTED; // Current benchmark state.
- private long mNanoPreviousTime = 0; // Previously captured System.nanoTime().
- private long mNanoFinishTime = 0; // Finish if System.nanoTime() returns after than this value.
- private long mNanoPausedTime = 0; // The System.nanoTime() when the pauseTiming() is called.
- private long mNanoPausedDuration = 0; // The duration of paused state in nano sec.
- private long mNanoTimeLimit = 1 * 1000 * 1000 * 1000; // 1 sec. Default time limit.
+ private static final long WARMUP_DURATION_NS = ms2ns(250); // warm-up for at least 250ms
+ private static final int WARMUP_MIN_ITERATIONS = 16; // minimum iterations to warm-up for
+
+ // TODO: Tune these values.
+ private static final long TARGET_TEST_DURATION_NS = ms2ns(500); // target testing for 500 ms
+ private static final int MAX_TEST_ITERATIONS = 1000000;
+ private static final int MIN_TEST_ITERATIONS = 100;
+ private static final int REPEAT_COUNT = 5;
+
+ private long mStartTimeNs = 0; // Previously captured System.nanoTime().
+ private long mPausedTimeNs = 0; // The System.nanoTime() when the pauseTiming() is called.
+ private long mPausedDurationNs = 0; // The duration of paused state in nano sec.
+
+ private int mIteration = 0;
+ private int mMaxIterations = 0;
+
+ private int mRepeatCount = 0;
// Statistics. These values will be filled when the benchmark has finished.
// The computation needs double precision, but long int is fine for final reporting.
private long mMedian = 0;
private double mMean = 0.0;
private double mStandardDeviation = 0.0;
-
- // Number of iterations needed for calculating the stats.
- private int mMinRepeatTimes = 16;
+ private long mMin = 0;
// Individual duration in nano seconds.
private ArrayList<Long> mResults = new ArrayList<>();
- /**
- * Sets the number of iterations needed for calculating the stats. Default is 16.
- */
- public void setMinRepeatTimes(int minRepeatTimes) {
- mMinRepeatTimes = minRepeatTimes;
+ private static final long ms2ns(long ms) {
+ return TimeUnit.MILLISECONDS.toNanos(ms);
}
/**
@@ -89,8 +98,13 @@
mMedian = size % 2 == 0 ? (mResults.get(size / 2) + mResults.get(size / 2 + 1)) / 2 :
mResults.get(size / 2);
+ mMin = mResults.get(0);
for (int i = 0; i < size; ++i) {
- mMean += mResults.get(i);
+ long result = mResults.get(i);
+ mMean += result;
+ if (result < mMin) {
+ mMin = result;
+ }
}
mMean /= (double) size;
@@ -108,7 +122,7 @@
throw new IllegalStateException(
"Unable to pause the benchmark. The benchmark has already paused.");
}
- mNanoPausedTime = System.nanoTime();
+ mPausedTimeNs = System.nanoTime();
mState = RUNNING_PAUSED;
}
@@ -119,11 +133,43 @@
throw new IllegalStateException(
"Unable to resume the benchmark. The benchmark is already running.");
}
- mNanoPausedDuration += System.nanoTime() - mNanoPausedTime;
- mNanoPausedTime = 0;
+ mPausedDurationNs += System.nanoTime() - mPausedTimeNs;
+ mPausedTimeNs = 0;
mState = RUNNING;
}
+ private void beginWarmup() {
+ mStartTimeNs = System.nanoTime();
+ mIteration = 0;
+ mState = WARMUP;
+ }
+
+ private void beginBenchmark(long warmupDuration, int iterations) {
+ mMaxIterations = (int) (TARGET_TEST_DURATION_NS / (warmupDuration / iterations));
+ mMaxIterations = Math.min(MAX_TEST_ITERATIONS,
+ Math.max(mMaxIterations, MIN_TEST_ITERATIONS));
+ mPausedDurationNs = 0;
+ mIteration = 0;
+ mRepeatCount = 0;
+ mState = RUNNING;
+ mStartTimeNs = System.nanoTime();
+ }
+
+ private boolean startNextTestRun() {
+ final long currentTime = System.nanoTime();
+ mResults.add((currentTime - mStartTimeNs - mPausedDurationNs) / mMaxIterations);
+ mRepeatCount++;
+ if (mRepeatCount >= REPEAT_COUNT) {
+ calculateSatistics();
+ mState = FINISHED;
+ return false;
+ }
+ mPausedDurationNs = 0;
+ mIteration = 0;
+ mStartTimeNs = System.nanoTime();
+ return true;
+ }
+
/**
* Judges whether the benchmark needs more samples.
*
@@ -132,23 +178,22 @@
public boolean keepRunning() {
switch (mState) {
case NOT_STARTED:
- mNanoPreviousTime = System.nanoTime();
- mNanoFinishTime = mNanoPreviousTime + mNanoTimeLimit;
- mState = RUNNING;
+ beginWarmup();
+ return true;
+ case WARMUP:
+ mIteration++;
+ // Only check nanoTime on every iteration in WARMUP since we
+ // don't yet have a target iteration count.
+ final long duration = System.nanoTime() - mStartTimeNs;
+ if (mIteration >= WARMUP_MIN_ITERATIONS && duration >= WARMUP_DURATION_NS) {
+ beginBenchmark(duration, mIteration);
+ }
return true;
case RUNNING:
- final long currentTime = System.nanoTime();
- mResults.add(currentTime - mNanoPreviousTime - mNanoPausedDuration);
- mNanoPausedDuration = 0;
-
- // To calculate statistics, needs two or more samples.
- if (mResults.size() > mMinRepeatTimes && currentTime > mNanoFinishTime) {
- calculateSatistics();
- mState = FINISHED;
- return false;
+ mIteration++;
+ if (mIteration >= mMaxIterations) {
+ return startNextTestRun();
}
-
- mNanoPreviousTime = currentTime;
return true;
case RUNNING_PAUSED:
throw new IllegalStateException(
@@ -161,21 +206,28 @@
}
}
- public long mean() {
+ private long mean() {
if (mState != FINISHED) {
throw new IllegalStateException("The benchmark hasn't finished");
}
return (long) mMean;
}
- public long median() {
+ private long median() {
if (mState != FINISHED) {
throw new IllegalStateException("The benchmark hasn't finished");
}
return mMedian;
}
- public long standardDeviation() {
+ private long min() {
+ if (mState != FINISHED) {
+ throw new IllegalStateException("The benchmark hasn't finished");
+ }
+ return mMin;
+ }
+
+ private long standardDeviation() {
if (mState != FINISHED) {
throw new IllegalStateException("The benchmark hasn't finished");
}
@@ -187,10 +239,11 @@
sb.append("Summary: ");
sb.append("median=").append(median()).append("ns, ");
sb.append("mean=").append(mean()).append("ns, ");
+ sb.append("min=").append(min()).append("ns, ");
sb.append("sigma=").append(standardDeviation()).append(", ");
sb.append("iteration=").append(mResults.size()).append(", ");
// print out the first few iterations' number for double checking.
- int sampleNumber = Math.min(mResults.size(), mMinRepeatTimes);
+ int sampleNumber = Math.min(mResults.size(), 16);
for (int i = 0; i < sampleNumber; i++) {
sb.append("No ").append(i).append(" result is ").append(mResults.get(i)).append(", ");
}
@@ -202,6 +255,7 @@
Bundle status = new Bundle();
status.putLong(key + "_median", median());
status.putLong(key + "_mean", mean());
+ status.putLong(key + "_min", min());
status.putLong(key + "_standardDeviation", standardDeviation());
instrumentation.sendStatus(Activity.RESULT_OK, status);
}