Initial checkin of GM verifier framework
The goal of the verifier framework is to enable opt-in checks of the
images produced by individual GMs. The basis of verification will be
comparing the rendered output of a GM against a source-of-truth image,
such as one generated by the CPU backend.
In the short term this can enable coarse-grained sanity checks for a
subset of GMs to catch e.g. egregious rendering bugs. In the longer term
this can provide an SkQP-style suite of tests that can be run across
many/all GMs to provide a vote of confidence in the rendering
correctness of new devices.
Bug: skia:9855
Change-Id: Id7310de8005ffa7e8eb2fd0e4008f5f8db1419ab
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/267761
Commit-Queue: Derek Sollenberger <djsollen@google.com>
Auto-Submit: Tyler Denniston <tdenniston@google.com>
Reviewed-by: Derek Sollenberger <djsollen@google.com>
diff --git a/dm/DM.cpp b/dm/DM.cpp
index 02261c0..1914561 100644
--- a/dm/DM.cpp
+++ b/dm/DM.cpp
@@ -7,6 +7,7 @@
#include "dm/DMJsonWriter.h"
#include "dm/DMSrcSink.h"
+#include "gm/verifiers/gmverifier.h"
#include "include/codec/SkCodec.h"
#include "include/core/SkBBHFactory.h"
#include "include/core/SkColorPriv.h"
@@ -150,6 +151,9 @@
static DEFINE_bool(rasterize_pdf, false, "Rasterize PDFs when possible.");
+static DEFINE_bool(runVerifiers, false,
+ "if true, run SkQP-style verification of GM-produced images.");
+
#if defined(__MSVC_RUNTIME_CHECKS)
#include <rtcapi.h>
@@ -1132,6 +1136,11 @@
}
}
+ // Run verifiers if specified
+ if (FLAGS_runVerifiers) {
+ RunGMVerifiers(task, bitmap);
+ }
+
// We're likely switching threads here, so we must capture by value, [=] or [foo,bar].
SkStreamAsset* data = stream.detachAsStream().release();
gDefinitelyThreadSafeWork.add([task,name,bitmap,data]{
@@ -1388,6 +1397,23 @@
}
}
}
+
+ static void RunGMVerifiers(const Task& task, const SkBitmap& actualBitmap) {
+ const SkString name = task.src->name();
+ auto verifierList = task.src->getVerifiers();
+ if (verifierList == nullptr) {
+ return;
+ }
+
+ skiagm::verifiers::VerifierResult
+ res = verifierList->verifyAll(task.sink->colorInfo(), actualBitmap);
+ if (!res.ok()) {
+ fail(
+ SkStringPrintf(
+ "%s %s %s %s: verifier failed: %s", task.sink.tag.c_str(), task.src.tag.c_str(),
+ task.src.options.c_str(), name.c_str(), res.message().c_str()));
+ }
+ }
};
/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
diff --git a/dm/DMSrcSink.cpp b/dm/DMSrcSink.cpp
index 22fe5e9..da68301 100644
--- a/dm/DMSrcSink.cpp
+++ b/dm/DMSrcSink.cpp
@@ -6,6 +6,7 @@
*/
#include "dm/DMSrcSink.h"
+#include "gm/verifiers/gmverifier.h"
#include "include/codec/SkAndroidCodec.h"
#include "include/codec/SkCodec.h"
#include "include/core/SkColorSpace.h"
@@ -108,6 +109,11 @@
gm->modifyGrContextOptions(options);
}
+std::unique_ptr<skiagm::verifiers::VerifierList> GMSrc::getVerifiers() const {
+ std::unique_ptr<skiagm::GM> gm(fFactory());
+ return gm->getVerifiers();
+}
+
/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
BRDSrc::BRDSrc(Path path, Mode mode, CodecSrc::DstColorType dstColorType, uint32_t sampleSize)
diff --git a/dm/DMSrcSink.h b/dm/DMSrcSink.h
index 63b3f7f..174f69d 100644
--- a/dm/DMSrcSink.h
+++ b/dm/DMSrcSink.h
@@ -24,6 +24,12 @@
//#define TEST_VIA_SVG
+namespace skiagm {
+namespace verifiers {
+class VerifierList;
+}
+}
+
namespace DM {
// This is just convenience. It lets you use either return "foo" or return SkStringPrintf(...).
@@ -82,6 +88,11 @@
virtual SkISize size(int) const { return this->size(); }
// Force Tasks using this Src to run on the main thread?
virtual bool serial() const { return false; }
+
+ /** Return a list of verifiers for the src, or null if no verifiers should be run .*/
+ virtual std::unique_ptr<skiagm::verifiers::VerifierList> getVerifiers() const {
+ return nullptr;
+ }
};
struct Sink {
@@ -97,6 +108,9 @@
virtual const char* fileExtension() const = 0;
virtual SinkFlags flags() const = 0;
+
+ /** Returns the color type and space used by the sink. */
+ virtual SkColorInfo colorInfo() const { return SkColorInfo(); }
};
/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
@@ -110,6 +124,8 @@
Name name() const override;
void modifyGrContextOptions(GrContextOptions* options) const override;
+ std::unique_ptr<skiagm::verifiers::VerifierList> getVerifiers() const override;
+
private:
skiagm::GMFactory fFactory;
};
@@ -351,6 +367,9 @@
return SinkFlags{ SinkFlags::kGPU, SinkFlags::kDirect, ms };
}
const GrContextOptions& baseContextOptions() const { return fBaseContextOptions; }
+ SkColorInfo colorInfo() const override {
+ return SkColorInfo(fColorType, fAlphaType, fColorSpace);
+ }
private:
sk_gpu_test::GrContextFactory::ContextType fContextType;
diff --git a/gm/gm.cpp b/gm/gm.cpp
index b196b91..4a49c2a 100644
--- a/gm/gm.cpp
+++ b/gm/gm.cpp
@@ -6,6 +6,7 @@
*/
#include "gm/gm.h"
+#include "gm/verifiers/gmverifier.h"
#include "include/core/SkBitmap.h"
#include "include/core/SkBlendMode.h"
#include "include/core/SkCanvas.h"
@@ -143,6 +144,11 @@
bool GM::runAsBench() const { return false; }
void GM::modifyGrContextOptions(GrContextOptions* options) {}
+std::unique_ptr<verifiers::VerifierList> GM::getVerifiers() const {
+ // No verifiers by default.
+ return nullptr;
+}
+
void GM::onOnceBeforeDraw() {}
bool GM::onAnimate(double /*nanos*/) { return false; }
diff --git a/gm/gm.h b/gm/gm.h
index 4d5bb0b..1bebe6b 100644
--- a/gm/gm.h
+++ b/gm/gm.h
@@ -8,6 +8,7 @@
#ifndef skiagm_DEFINED
#define skiagm_DEFINED
+#include "gm/verifiers/gmverifier.h"
#include "include/core/SkColor.h"
#include "include/core/SkScalar.h"
#include "include/core/SkSize.h"
@@ -148,6 +149,8 @@
virtual void modifyGrContextOptions(GrContextOptions*);
+ virtual std::unique_ptr<verifiers::VerifierList> getVerifiers() const;
+
protected:
virtual void onOnceBeforeDraw();
virtual DrawResult onDraw(SkCanvas* canvas, SkString* errorMsg);
@@ -176,6 +179,12 @@
class GpuGM : public GM {
public:
GpuGM(SkColor backgroundColor = SK_ColorWHITE) : GM(backgroundColor) {}
+
+ // TODO(tdenniston): Currently GpuGMs don't have verifiers (because they do not render on
+ // CPU), but we may want to be able to verify the output images standalone, without
+ // requiring a gold image for comparison.
+ std::unique_ptr<verifiers::VerifierList> getVerifiers() const override { return nullptr; }
+
private:
using GM::onDraw;
DrawResult onDraw(SkCanvas*, SkString* errorMsg) final;
diff --git a/gm/verifiers/gmverifier.cpp b/gm/verifiers/gmverifier.cpp
new file mode 100644
index 0000000..4f92b77
--- /dev/null
+++ b/gm/verifiers/gmverifier.cpp
@@ -0,0 +1,128 @@
+/*
+ * Copyright 2020 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include "gm/gm.h"
+#include "gm/verifiers/gmverifier.h"
+#include "include/core/SkBitmap.h"
+#include "include/core/SkCanvas.h"
+#include "include/core/SkSurface.h"
+#include "include/effects/SkImageFilters.h"
+#include "include/encode/SkPngEncoder.h"
+#include "src/utils/SkOSPath.h"
+
+/** Checks the given VerifierResult. If it is not ok, returns it. */
+#define RETURN_NOT_OK(res) if (!(res).ok()) return (res)
+
+namespace skiagm {
+namespace verifiers {
+
+VerifierResult::VerifierResult() : VerifierResult(Code::kOk, SkString("Ok")) {}
+
+VerifierResult::VerifierResult(VerifierResult::Code code, const SkString& msg) :
+ fCode(code), fMessage(msg) {}
+
+bool VerifierResult::ok() const {
+ return fCode == Code::kOk;
+}
+
+const SkString& VerifierResult::message() const {
+ return fMessage;
+}
+
+VerifierResult VerifierResult::Ok() {
+ return VerifierResult(Code::kOk, SkString("Ok"));
+}
+
+VerifierResult VerifierResult::Fail(const SkString& msg) {
+ return VerifierResult(Code::kFail, msg);
+}
+
+GMVerifier::GMVerifier(InputType inputType) : fInputType(inputType) {}
+
+GMVerifier::~GMVerifier() {}
+
+bool GMVerifier::needsGoldImage() const {
+ return fInputType == InputType::kGoldImageRequired;
+}
+
+VerifierResult GMVerifier::verify(const SkBitmap& gold, const SkBitmap& actual) {
+ // Call into specific implementation.
+ return verifyWithGold(actual.bounds(), gold, actual);
+}
+
+VerifierResult GMVerifier::verify(const SkBitmap& actual) {
+ // Call into specific implementation.
+ return verify(actual.bounds(), actual);
+}
+
+SkBitmap GMVerifier::RenderGoldBmp(skiagm::GM* gm, const SkColorInfo& colorInfo) {
+ SkASSERT(gm);
+
+ // Call into the GM instance to get the initial image.
+ const SkISize size = gm->getISize();
+ SkBitmap goldBmp;
+ goldBmp.allocPixels(SkImageInfo::Make(size, colorInfo));
+ SkCanvas canvas(goldBmp);
+ gm->draw(&canvas);
+
+ // Convert into common verifier colorspace.
+ SkBitmap goldVerifierBmp;
+ goldVerifierBmp.allocPixels(SkImageInfo::Make(size, VerifierColorInfo()));
+ SkCanvas verifierCanvas(goldVerifierBmp);
+ verifierCanvas.drawBitmap(goldBmp, 0, 0);
+
+ return goldVerifierBmp;
+}
+
+SkColorInfo GMVerifier::VerifierColorInfo() {
+ return SkColorInfo(
+ kRGBA_F16_SkColorType, kPremul_SkAlphaType,
+ SkColorSpace::MakeRGB(SkNamedTransferFn::kRec2020, SkNamedGamut::kRec2020));
+}
+
+VerifierResult GMVerifier::makeError(const SkString& msg) const {
+ return VerifierResult::Fail(SkStringPrintf("[%s] %s", name().c_str(), msg.c_str()));
+}
+
+VerifierList::VerifierList(GM* gm) : fGM(gm), fFailedVerifier(nullptr) {}
+
+void VerifierList::add(std::unique_ptr<GMVerifier> verifier) {
+ fVerifiers.push_back(std::move(verifier));
+}
+
+bool VerifierList::needsGoldImage() const {
+ for (const auto& v : fVerifiers) {
+ if (v->needsGoldImage()) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+VerifierResult VerifierList::verifyAll(const SkColorInfo& colorInfo, const SkBitmap& actual) {
+ // Render the golden image if any verifiers need it.
+ SkBitmap goldBmp;
+ if (needsGoldImage()) {
+ goldBmp = GMVerifier::RenderGoldBmp(fGM, colorInfo);
+ }
+
+ for (const auto& v : fVerifiers) {
+ fFailedVerifier = v.get();
+ if (v->needsGoldImage()) {
+ RETURN_NOT_OK(v->verify(goldBmp, actual));
+ } else {
+ RETURN_NOT_OK(v->verify(actual));
+ }
+ }
+
+ fFailedVerifier = nullptr;
+ return VerifierResult::Ok();
+}
+
+}
+}
diff --git a/gm/verifiers/gmverifier.h b/gm/verifiers/gmverifier.h
new file mode 100644
index 0000000..0bbd7ff
--- /dev/null
+++ b/gm/verifiers/gmverifier.h
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2020 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#ifndef gmverifier_DEFINED
+#define gmverifier_DEFINED
+
+#include "include/core/SkColor.h"
+#include "include/core/SkRect.h"
+#include "include/core/SkString.h"
+
+#include <vector>
+
+class SkBitmap;
+
+namespace skiagm {
+
+class GM;
+
+namespace verifiers {
+
+/** Result type for GM verifiers. */
+class VerifierResult {
+public:
+ VerifierResult();
+
+ /** Returns true if the result is ok (non-error). */
+ bool ok() const;
+
+ /** Returns reference to any message associated with the result. */
+ const SkString& message() const;
+
+ /** Constructs an "ok" (non-error) result. */
+ static VerifierResult Ok();
+
+ /** Constructs a "fail" (error) result with a specific message. */
+ static VerifierResult Fail(const SkString& msg);
+
+private:
+ /** Underlying error code. */
+ enum class Code {
+ kOk, kFail
+ };
+
+ /** Result code */
+ Code fCode;
+
+ /** Result message (may be empty). */
+ SkString fMessage;
+
+ /** Private constructor for a result with a specific code and message. */
+ VerifierResult(Code code, const SkString& msg);
+};
+
+/**
+ * Abstract base class for GM verifiers. A verifier checks the rendered output image of a GM.
+ *
+ * Different verifiers perform different types of transforms and checks. Verifiers may check the
+ * output of a GM against a given "golden" image which represents the correct output, or just
+ * check the output image of the GM by itself.
+ *
+ * Most verifiers have configurable fuzziness in the comparisons performed against the golden image.
+ *
+ * Subclasses should inherit from one of StandaloneVerifier or GoldImageVerifier instead of
+ * directly from this base class.
+ */
+class GMVerifier {
+public:
+ GMVerifier() = delete;
+
+ virtual ~GMVerifier();
+
+ /** Returns the human-friendly name of the verifier. */
+ virtual SkString name() const = 0;
+
+ /** Returns true if this verifier needs the gold image as input. */
+ bool needsGoldImage() const;
+
+ /**
+ * Runs the verifier. This method should be used if the verifier needs the gold image as input.
+ *
+ * @param gold Bitmap containing the "correct" image.
+ * @param actual Bitmap containing rendered output of a GM.
+ * @return Ok if the verification passed, or an error if not.
+ */
+ VerifierResult verify(const SkBitmap& gold, const SkBitmap& actual);
+
+ /**
+ * Runs the verifier.
+ *
+ * @param actual Bitmap containing rendered output of a GM.
+ * @return Ok if the verification passed, or an error if not.
+ */
+ VerifierResult verify(const SkBitmap& actual);
+
+ /** Renders the GM using the "golden" configuration. This is common across all GMs/verifiers. */
+ static SkBitmap RenderGoldBmp(skiagm::GM* gm, const SkColorInfo& colorInfo);
+
+ /**
+ * Gets the color information that all verifier inputs should be transformed into.
+ *
+ * The primary reason for having a single shared colorspace/color type is making per-pixel
+ * comparisons easier. Both the image under test and gold image are transformed into a shared
+ * colorspace which allows for getting per-pixel colors in SkColor4f.
+ */
+ static SkColorInfo VerifierColorInfo();
+
+protected:
+ /** The type of input required for the verifier. */
+ enum class InputType {
+ kGoldImageRequired, kStandalone
+ };
+
+ /** Set depending if the verifier needs a golden image as an input. */
+ InputType fInputType;
+
+ /** Constructor. */
+ GMVerifier(InputType inputType);
+
+ /** Implementation of the verification. */
+ virtual VerifierResult verifyWithGold(
+ const SkIRect& region, const SkBitmap& gold, const SkBitmap& actual) = 0;
+
+ /** Implementation of the verification. */
+ virtual VerifierResult verify(const SkIRect& region, const SkBitmap& actual) = 0;
+
+ /** Returns an error result formatted appropriately. */
+ VerifierResult makeError(const SkString& msg) const;
+};
+
+/**
+ * A verifier that operates standalone on the given input image (no comparison against a golden
+ * image).
+ */
+class StandaloneVerifier : public GMVerifier {
+public:
+ StandaloneVerifier() : GMVerifier(InputType::kStandalone) {}
+
+protected:
+ VerifierResult verifyWithGold(const SkIRect&, const SkBitmap&, const SkBitmap&) final {
+ return makeError(SkString("Verifier does not accept gold image input"));
+ }
+};
+
+/**
+ * A verifier that operates compares input image against a golden image.
+ */
+class GoldImageVerifier : public GMVerifier {
+public:
+ GoldImageVerifier() : GMVerifier(InputType::kGoldImageRequired) {}
+
+protected:
+ VerifierResult verify(const SkIRect&, const SkBitmap&) final {
+ return makeError(SkString("Verifier does not accept standalone input"));
+ }
+};
+
+/** A list of GM verifiers. */
+class VerifierList {
+public:
+ /** Constructs a VerifierList with the given gm instance. */
+ explicit VerifierList(GM* gm);
+
+ /** Adds a verifier to the list of verifiers. */
+ void add(std::unique_ptr<GMVerifier> verifier);
+
+ /**
+ * Runs all verifiers against the given input. If any verifiers fail, returns the first error.
+ * Else, returns ok. This version can be used if no verifiers in the list require the gold
+ * image as input.
+ */
+ VerifierResult verifyAll(const SkColorInfo& colorInfo, const SkBitmap& actual);
+
+private:
+ /** The parent GM instance of this VerifierList. */
+ GM* fGM;
+
+ /** The list of verifiers. */
+ std::vector<std::unique_ptr<GMVerifier>> fVerifiers;
+
+ /** After running, set to the first verifier that failed, or nullptr if none failed. */
+ const GMVerifier* fFailedVerifier;
+
+ /** Returns true if any verifiers in the list need the gold image as input. */
+ bool needsGoldImage() const;
+};
+
+}
+}
+
+#endif
diff --git a/gn/gm.gni b/gn/gm.gni
index fbe83d1..d24fcb6 100644
--- a/gn/gm.gni
+++ b/gn/gm.gni
@@ -389,6 +389,7 @@
"$_gm/xfermodes3.cpp",
"$_gm/yuvtorgbeffect.cpp",
"$_gm/yuv420_odd_dim.cpp",
+ "$_gm/verifiers/gmverifier.cpp",
]
gl_gm_sources = [