Improve perf testing framework
The ANGLERenderTest subclasses have a parameter "iterations". Previously
most of these tests would perform iterations^2 iterations, since the
looping was done both in ANGLERenderTest and in the individual test
classes. Do the looping only in the individual test classes instead.
This enables getting rid of separate beginDrawBenchmark() and
endDrawBenchmark() functions.
Some other unused code is also removed:
1. stepBenchmark function
2. unused parameters to step()
This makes the core loop of running tests simpler.
The perf testing framework also now has shared logic for deciding when
to end a given test.
The score calculation for tests is also changed. Instead of reporting
just the number of operations done, it is reported relative to the actual
run time of the test. This should make the test results more accurate,
since run time of the tests may have some variation. It also enables
changing the run time of the tests without rebaselining them.
In the tests that use GPU, GPU operations are also waited to finish
before stopping the timer.
BUG=angleproject:1261
TEST=angle_perftests
Change-Id: I69e9aad8afd2d9dedd60e144f0a5d4203618feef
Reviewed-on: https://chromium-review.googlesource.com/319381
Tryjob-Request: Olli Etuaho <oetuaho@nvidia.com>
Tested-by: Olli Etuaho <oetuaho@nvidia.com>
Reviewed-by: Jamie Madill <jmadill@chromium.org>
Reviewed-by: Zhenyao Mo <zmo@chromium.org>
diff --git a/src/tests/perf_tests/ANGLEPerfTest.cpp b/src/tests/perf_tests/ANGLEPerfTest.cpp
index a8b8f50..588b7b1 100644
--- a/src/tests/perf_tests/ANGLEPerfTest.cpp
+++ b/src/tests/perf_tests/ANGLEPerfTest.cpp
@@ -14,9 +14,10 @@
ANGLEPerfTest::ANGLEPerfTest(const std::string &name, const std::string &suffix)
: mName(name),
mSuffix(suffix),
- mRunning(false),
mTimer(nullptr),
- mNumFrames(0)
+ mNumStepsPerformed(0),
+ mRunTimeSeconds(5.0),
+ mRunning(true)
{
mTimer = CreateTimer();
}
@@ -29,23 +30,20 @@
void ANGLEPerfTest::run()
{
mTimer->start();
- double prevTime = 0.0;
-
while (mRunning)
{
- double elapsedTime = mTimer->getElapsedTime();
- double deltaTime = elapsedTime - prevTime;
-
- ++mNumFrames;
- step(static_cast<float>(deltaTime), elapsedTime);
-
- if (!mRunning)
+ step();
+ if (mRunning)
{
- break;
+ ++mNumStepsPerformed;
}
-
- prevTime = elapsedTime;
+ if (mTimer->getElapsedTime() > mRunTimeSeconds)
+ {
+ mRunning = false;
+ }
}
+ finishTest();
+ mTimer->stop();
}
void ANGLEPerfTest::printResult(const std::string &trace, double value, const std::string &units, bool important) const
@@ -60,17 +58,17 @@
void ANGLEPerfTest::SetUp()
{
- mRunning = true;
}
void ANGLEPerfTest::TearDown()
{
- printResult("score", static_cast<size_t>(mNumFrames), "score", true);
+ double relativeScore = static_cast<double>(mNumStepsPerformed) / mTimer->getElapsedTime();
+ printResult("score", static_cast<size_t>(std::round(relativeScore)), "score", true);
}
double ANGLEPerfTest::normalizedTime(size_t value) const
{
- return static_cast<double>(value) / static_cast<double>(mNumFrames);
+ return static_cast<double>(value) / static_cast<double>(mNumStepsPerformed);
}
std::string RenderTestParams::suffix() const
@@ -96,8 +94,6 @@
ANGLERenderTest::ANGLERenderTest(const std::string &name, const RenderTestParams &testParams)
: ANGLEPerfTest(name, testParams.suffix()),
mTestParams(testParams),
- mDrawIterations(10),
- mRunTimeSeconds(5.0),
mEGLWindow(nullptr),
mOSWindow(nullptr)
{
@@ -143,47 +139,44 @@
mOSWindow->destroy();
}
-void ANGLERenderTest::step(float dt, double totalTime)
+void ANGLERenderTest::step()
{
- stepBenchmark(dt, totalTime);
-
// Clear events that the application did not process from this frame
Event event;
+ bool closed = false;
while (popEvent(&event))
{
// If the application did not catch a close event, close now
if (event.Type == Event::EVENT_CLOSED)
{
- mRunning = false;
+ closed = true;
}
}
- if (mRunning)
+ if (closed)
{
- draw();
- mEGLWindow->swap();
+ abortTest();
+ }
+ else
+ {
+ drawBenchmark();
+ // Swap is needed so that the GPU driver will occasionally flush its internal command queue
+ // to the GPU. The null device benchmarks are only testing CPU overhead, so they don't need
+ // to swap.
+ if (mTestParams.eglParameters.deviceType != EGL_PLATFORM_ANGLE_DEVICE_TYPE_NULL_ANGLE)
+ {
+ mEGLWindow->swap();
+ }
mOSWindow->messageLoop();
}
}
-void ANGLERenderTest::draw()
+void ANGLERenderTest::finishTest()
{
- if (mTimer->getElapsedTime() > mRunTimeSeconds)
+ if (mTestParams.eglParameters.deviceType != EGL_PLATFORM_ANGLE_DEVICE_TYPE_NULL_ANGLE)
{
- mRunning = false;
- return;
+ glFinish();
}
-
- ++mNumFrames;
-
- beginDrawBenchmark();
-
- for (unsigned int iteration = 0; iteration < mDrawIterations; ++iteration)
- {
- drawBenchmark();
- }
-
- endDrawBenchmark();
}
bool ANGLERenderTest::popEvent(Event *event)
diff --git a/src/tests/perf_tests/ANGLEPerfTest.h b/src/tests/perf_tests/ANGLEPerfTest.h
index 301f564..e0b999a 100644
--- a/src/tests/perf_tests/ANGLEPerfTest.h
+++ b/src/tests/perf_tests/ANGLEPerfTest.h
@@ -36,7 +36,10 @@
ANGLEPerfTest(const std::string &name, const std::string &suffix);
virtual ~ANGLEPerfTest();
- virtual void step(float dt, double totalTime) = 0;
+ virtual void step() = 0;
+
+ // Called right before timer is stopped to let the test wait for asynchronous operations.
+ virtual void finishTest() {}
protected:
void run();
@@ -48,12 +51,19 @@
// Normalize a time value according to the number of test loop iterations (mFrameCount)
double normalizedTime(size_t value) const;
+ // Call if the test step was aborted and the test should stop running.
+ void abortTest() { mRunning = false; }
+
+ int getNumStepsPerformed() const { return mNumStepsPerformed; }
+
std::string mName;
std::string mSuffix;
-
- bool mRunning;
Timer *mTimer;
- int mNumFrames;
+ double mRunTimeSeconds;
+
+ private:
+ int mNumStepsPerformed;
+ bool mRunning;
};
struct RenderTestParams : public angle::PlatformParameters
@@ -73,11 +83,7 @@
virtual void initializeBenchmark() { }
virtual void destroyBenchmark() { }
- virtual void stepBenchmark(float dt, double totalTime) { }
-
- virtual void beginDrawBenchmark() { }
virtual void drawBenchmark() = 0;
- virtual void endDrawBenchmark() { }
bool popEvent(Event *event);
@@ -85,15 +91,13 @@
protected:
const RenderTestParams &mTestParams;
- unsigned int mDrawIterations;
- double mRunTimeSeconds;
private:
void SetUp() override;
void TearDown() override;
- void step(float dt, double totalTime) override;
- void draw();
+ void step() override;
+ void finishTest() override;
EGLWindow *mEGLWindow;
OSWindow *mOSWindow;
diff --git a/src/tests/perf_tests/BufferSubData.cpp b/src/tests/perf_tests/BufferSubData.cpp
index e5f436b..5818d52 100644
--- a/src/tests/perf_tests/BufferSubData.cpp
+++ b/src/tests/perf_tests/BufferSubData.cpp
@@ -28,7 +28,7 @@
windowHeight = 512;
updateSize = 3000;
bufferSize = 40000000;
- iterations = 2;
+ iterations = 4;
updateRate = 1;
}
@@ -59,7 +59,6 @@
void initializeBenchmark() override;
void destroyBenchmark() override;
- void beginDrawBenchmark() override;
void drawBenchmark() override;
private:
@@ -269,7 +268,6 @@
ASSERT_LT(1, params.vertexComponentCount);
ASSERT_LT(0u, params.iterations);
- mDrawIterations = params.iterations;
const std::string vs = SHADER_SOURCE
(
@@ -359,19 +357,15 @@
SafeDeleteArray(mUpdateData);
}
-void BufferSubDataBenchmark::beginDrawBenchmark()
-{
- // Clear the color buffer
- glClear(GL_COLOR_BUFFER_BIT);
-}
-
void BufferSubDataBenchmark::drawBenchmark()
{
+ glClear(GL_COLOR_BUFFER_BIT);
+
const auto ¶ms = GetParam();
for (unsigned int it = 0; it < params.iterations; it++)
{
- if (params.updateSize > 0 && ((mNumFrames % params.updateRate) == 0))
+ if (params.updateSize > 0 && ((getNumStepsPerformed() % params.updateRate) == 0))
{
glBufferSubData(GL_ARRAY_BUFFER, 0, params.updateSize, mUpdateData);
}
diff --git a/src/tests/perf_tests/DrawCallPerf.cpp b/src/tests/perf_tests/DrawCallPerf.cpp
index a07905a..b8c82d5 100644
--- a/src/tests/perf_tests/DrawCallPerf.cpp
+++ b/src/tests/perf_tests/DrawCallPerf.cpp
@@ -69,7 +69,6 @@
void initializeBenchmark() override;
void destroyBenchmark() override;
- void beginDrawBenchmark() override;
void drawBenchmark() override;
private:
@@ -92,7 +91,6 @@
const auto ¶ms = GetParam();
ASSERT_LT(0u, params.iterations);
- mDrawIterations = params.iterations;
const std::string vs = SHADER_SOURCE
(
@@ -166,14 +164,10 @@
glDeleteBuffers(1, &mBuffer);
}
-void DrawCallPerfBenchmark::beginDrawBenchmark()
-{
- // Clear the color buffer
- glClear(GL_COLOR_BUFFER_BIT);
-}
-
void DrawCallPerfBenchmark::drawBenchmark()
{
+ glClear(GL_COLOR_BUFFER_BIT);
+
const auto ¶ms = GetParam();
for (unsigned int it = 0; it < params.iterations; it++)
@@ -211,7 +205,7 @@
{
DrawCallPerfParams params;
params.eglParameters = DEFAULT();
- params.iterations = 100;
+ params.iterations = 10000;
params.numTris = 0;
params.runTimeSeconds = 5.0;
return params;
diff --git a/src/tests/perf_tests/EGLInitializePerf.cpp b/src/tests/perf_tests/EGLInitializePerf.cpp
index adb5a3b..da71ee1 100644
--- a/src/tests/perf_tests/EGLInitializePerf.cpp
+++ b/src/tests/perf_tests/EGLInitializePerf.cpp
@@ -77,7 +77,7 @@
EGLInitializePerfTest();
~EGLInitializePerfTest();
- void step(float dt, double totalTime) override;
+ void step() override;
void TearDown() override;
private:
@@ -132,18 +132,13 @@
SafeDelete(mOSWindow);
}
-void EGLInitializePerfTest::step(float dt, double totalTime)
+void EGLInitializePerfTest::step()
{
ASSERT_NE(EGL_NO_DISPLAY, mDisplay);
EGLint majorVersion, minorVersion;
ASSERT_EQ(static_cast<EGLBoolean>(EGL_TRUE), eglInitialize(mDisplay, &majorVersion, &minorVersion));
ASSERT_EQ(static_cast<EGLBoolean>(EGL_TRUE), eglTerminate(mDisplay));
-
- if (mTimer->getElapsedTime() >= 5.0)
- {
- mRunning = false;
- }
}
void EGLInitializePerfTest::TearDown()
diff --git a/src/tests/perf_tests/IndexConversionPerf.cpp b/src/tests/perf_tests/IndexConversionPerf.cpp
index 8eebec0..39a81ca 100644
--- a/src/tests/perf_tests/IndexConversionPerf.cpp
+++ b/src/tests/perf_tests/IndexConversionPerf.cpp
@@ -50,7 +50,6 @@
void initializeBenchmark() override;
void destroyBenchmark() override;
- void beginDrawBenchmark() override;
void drawBenchmark() override;
void updateBufferData();
@@ -78,8 +77,6 @@
ASSERT_LT(0u, params.iterations);
ASSERT_LT(0u, params.numIndexTris);
- mDrawIterations = params.iterations;
-
const std::string vs = SHADER_SOURCE
(
attribute vec2 vPosition;
@@ -165,14 +162,10 @@
glDeleteBuffers(1, &mIndexBuffer);
}
-void IndexConversionPerfTest::beginDrawBenchmark()
-{
- // Clear the color buffer
- glClear(GL_COLOR_BUFFER_BIT);
-}
-
void IndexConversionPerfTest::drawBenchmark()
{
+ glClear(GL_COLOR_BUFFER_BIT);
+
const auto ¶ms = GetParam();
// Trigger an update to ensure we convert once a frame
@@ -197,7 +190,7 @@
params.minorVersion = 0;
params.windowWidth = 256;
params.windowHeight = 256;
- params.iterations = 15;
+ params.iterations = 225;
params.numIndexTris = 3000;
return params;
}
diff --git a/src/tests/perf_tests/IndexDataManagerTest.cpp b/src/tests/perf_tests/IndexDataManagerTest.cpp
index 40ca085..b6465c0 100644
--- a/src/tests/perf_tests/IndexDataManagerTest.cpp
+++ b/src/tests/perf_tests/IndexDataManagerTest.cpp
@@ -116,7 +116,7 @@
public:
IndexDataManagerPerfTest();
- void step(float dt, double totalTime) override;
+ void step() override;
rx::IndexDataManager mIndexDataManager;
GLsizei mIndexCount;
@@ -151,7 +151,7 @@
mIndexBuffer.bufferData(&indexData[0], indexData.size() * sizeof(GLushort), GL_STATIC_DRAW);
}
-void IndexDataManagerPerfTest::step(float dt, double totalTime)
+void IndexDataManagerPerfTest::step()
{
rx::TranslatedIndexData translatedIndexData;
rx::SourceIndexData sourceIndexData;
@@ -162,11 +162,6 @@
mIndexDataManager.prepareIndexData(GL_UNSIGNED_SHORT, mIndexCount, &mIndexBuffer, nullptr,
&translatedIndexData, &sourceIndexData, false);
}
-
- if (mTimer->getElapsedTime() >= 5.0)
- {
- mRunning = false;
- }
}
TEST_F(IndexDataManagerPerfTest, Run)
diff --git a/src/tests/perf_tests/InstancingPerf.cpp b/src/tests/perf_tests/InstancingPerf.cpp
index 30f4921..524387c 100644
--- a/src/tests/perf_tests/InstancingPerf.cpp
+++ b/src/tests/perf_tests/InstancingPerf.cpp
@@ -83,7 +83,6 @@
void initializeBenchmark() override;
void destroyBenchmark() override;
- void beginDrawBenchmark() override;
void drawBenchmark() override;
private:
@@ -106,7 +105,6 @@
const auto ¶ms = GetParam();
ASSERT_LT(0u, params.iterations);
- mDrawIterations = params.iterations;
const std::string vs =
"attribute vec2 aPosition;\n"
@@ -268,13 +266,10 @@
}
}
-void InstancingPerfBenchmark::beginDrawBenchmark()
-{
- glClear(GL_COLOR_BUFFER_BIT);
-}
-
void InstancingPerfBenchmark::drawBenchmark()
{
+ glClear(GL_COLOR_BUFFER_BIT);
+
const auto ¶ms = GetParam();
// Animatino makes the test more interesting visually, but also eats up many CPU cycles.
diff --git a/src/tests/perf_tests/InterleavedAttributeData.cpp b/src/tests/perf_tests/InterleavedAttributeData.cpp
index da761d9..15e2a5b 100644
--- a/src/tests/perf_tests/InterleavedAttributeData.cpp
+++ b/src/tests/perf_tests/InterleavedAttributeData.cpp
@@ -54,7 +54,6 @@
void initializeBenchmark() override;
void destroyBenchmark() override;
- void beginDrawBenchmark() override;
void drawBenchmark() override;
private:
@@ -149,15 +148,11 @@
}
}
-void InterleavedAttributeDataBenchmark::beginDrawBenchmark()
-{
- // Clear the color buffer
- glClear(GL_COLOR_BUFFER_BIT);
-}
-
void InterleavedAttributeDataBenchmark::drawBenchmark()
{
- for (size_t k = 0; k < 4; k++)
+ glClear(GL_COLOR_BUFFER_BIT);
+
+ for (size_t k = 0; k < 20; k++)
{
for (size_t i = 0; i < ArraySize(mPositionColorBuffer); i++)
{
diff --git a/src/tests/perf_tests/PointSprites.cpp b/src/tests/perf_tests/PointSprites.cpp
index 184c708..c59c5ea 100644
--- a/src/tests/perf_tests/PointSprites.cpp
+++ b/src/tests/perf_tests/PointSprites.cpp
@@ -29,7 +29,7 @@
minorVersion = 0;
windowWidth = 1280;
windowHeight = 720;
- iterations = 10;
+ iterations = 100;
count = 10;
size = 3.0f;
numVaryings = 3;
@@ -59,7 +59,6 @@
void initializeBenchmark() override;
void destroyBenchmark() override;
- void beginDrawBenchmark() override;
void drawBenchmark() override;
private:
@@ -87,7 +86,6 @@
{
const auto ¶ms = GetParam();
- mDrawIterations = params.iterations;
ASSERT_LT(0u, params.iterations);
std::stringstream vstrstr;
@@ -184,14 +182,10 @@
glDeleteBuffers(1, &mBuffer);
}
-void PointSpritesBenchmark::beginDrawBenchmark()
-{
- // Clear the color buffer
- glClear(GL_COLOR_BUFFER_BIT);
-}
-
void PointSpritesBenchmark::drawBenchmark()
{
+ glClear(GL_COLOR_BUFFER_BIT);
+
const auto ¶ms = GetParam();
for (unsigned int it = 0; it < params.iterations; it++)
diff --git a/src/tests/perf_tests/TexSubImage.cpp b/src/tests/perf_tests/TexSubImage.cpp
index a89004c..3e4b5cf 100644
--- a/src/tests/perf_tests/TexSubImage.cpp
+++ b/src/tests/perf_tests/TexSubImage.cpp
@@ -31,7 +31,7 @@
imageHeight = 1024;
subImageWidth = 64;
subImageHeight = 64;
- iterations = 3;
+ iterations = 9;
}
std::string suffix() const override;
@@ -58,7 +58,6 @@
void initializeBenchmark() override;
void destroyBenchmark() override;
- void beginDrawBenchmark() override;
void drawBenchmark() override;
private:
@@ -108,7 +107,6 @@
const auto ¶ms = GetParam();
assert(params.iterations > 0);
- mDrawIterations = params.iterations;
// Use tightly packed data
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
@@ -220,7 +218,7 @@
delete[] mPixels;
}
-void TexSubImageBenchmark::beginDrawBenchmark()
+void TexSubImageBenchmark::drawBenchmark()
{
// Set the viewport
glViewport(0, 0, getWindow()->getWidth(), getWindow()->getHeight());
@@ -251,10 +249,7 @@
glUniform1i(mSamplerLoc, 0);
ASSERT_GL_NO_ERROR();
-}
-void TexSubImageBenchmark::drawBenchmark()
-{
const auto ¶ms = GetParam();
for (unsigned int iteration = 0; iteration < params.iterations; ++iteration)