tools/skqp/src/skqp.cpp

/*
 * Copyright 2018 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include "tools/skqp/src/skqp.h"

#include "gm/gm.h"
#include "include/core/SkFontStyle.h"
#include "include/core/SkGraphics.h"
#include "include/core/SkStream.h"
#include "include/core/SkSurface.h"
#include "include/encode/SkPngEncoder.h"
#include "include/gpu/GrContext.h"
#include "include/gpu/GrContextOptions.h"
#include "include/private/SkImageInfoPriv.h"
#include "src/core/SkFontMgrPriv.h"
#include "src/core/SkOSFile.h"
#include "src/core/SkStreamPriv.h"
#include "src/utils/SkOSPath.h"
#include "tests/Test.h"
#include "tools/fonts/TestFontMgr.h"
#include "tools/gpu/gl/GLTestContext.h"
#include "tools/gpu/vk/VkTestContext.h"

#include <limits.h>
#include <algorithm>
#include <cinttypes>
#include <sstream>

#include "tools/skqp/src/skqp_model.h"

#define IMAGES_DIRECTORY_PATH "images"
#define PATH_MAX_PNG "max.png"
#define PATH_MIN_PNG "min.png"
#define PATH_IMG_PNG "image.png"
#define PATH_ERR_PNG "errors.png"
#define PATH_MODEL "model"

static constexpr char kRenderTestCSVReport[] = "out.csv";
static constexpr char kRenderTestReportPath[] = "report.html";
static constexpr char kRenderTestsPath[] = "skqp/rendertests.txt";
static constexpr char kUnitTestReportPath[] = "unit_tests.txt";
static constexpr char kUnitTestsPath[]   = "skqp/unittests.txt";

// Kind of like Python's readlines(), but without any allocation.
// Calls f() on each line.
// F is [](const char*, size_t) -> void
template <typename F>
static void readlines(const void* data, size_t size, F f) {
    const char* start = (const char*)data;
    const char* end = start + size;
    const char* ptr = start;
    while (ptr < end) {
        while (*ptr++ != '\n' && ptr < end) {}
        size_t len = ptr - start;
        f(start, len);
        start = ptr;
    }
}

static void get_unit_tests(SkQPAssetManager* mgr, std::vector<SkQP::UnitTest>* unitTests) {
    std::unordered_set<std::string> testset;
    auto insert = [&testset](const char* s, size_t l) {
        SkASSERT(l > 1) ;
        if (l > 0 && s[l - 1] == '\n') {  // strip line endings.
            --l;
        }
        if (l > 0) {  // only add non-empty strings.
            testset.insert(std::string(s, l));
        }
    };
    if (sk_sp<SkData> dat = mgr->open(kUnitTestsPath)) {
        readlines(dat->data(), dat->size(), insert);
    }
    for (const skiatest::Test& test : skiatest::TestRegistry::Range()) {
        if ((testset.empty() || testset.count(std::string(test.name)) > 0) && test.needsGpu) {
            unitTests->push_back(&test);
        }
    }
    auto lt = [](SkQP::UnitTest u, SkQP::UnitTest v) { return strcmp(u->name, v->name) < 0; };
    std::sort(unitTests->begin(), unitTests->end(), lt);
}

static void get_render_tests(SkQPAssetManager* mgr,
                             std::vector<SkQP::GMFactory>* gmlist,
                             std::unordered_map<std::string, int64_t>* gmThresholds) {
    auto insert = [gmThresholds](const char* s, size_t l) {
        SkASSERT(l > 1) ;
        if (l > 0 && s[l - 1] == '\n') {  // strip line endings.
            --l;
        }
        if (l == 0) {
            return;
        }
        const char* end = s + l;
        const char* ptr = s;
        constexpr char kDelimeter = ',';
        while (ptr < end && *ptr != kDelimeter) { ++ptr; }
        if (ptr + 1 >= end) {
            SkASSERT(false);  // missing delimeter
            return;
        }
        std::string key(s, ptr - s);
        ++ptr;  // skip delimeter
        std::string number(ptr, end - ptr);  // null-terminated copy.
        int64_t value = 0;
        if (1 != sscanf(number.c_str(), "%" SCNd64 , &value)) {
            SkASSERT(false);  // Not a number
            return;
        }
        gmThresholds->insert({std::move(key), value});  // (*gmThresholds)[s] = value;
    };
    if (sk_sp<SkData> dat = mgr->open(kRenderTestsPath)) {
        readlines(dat->data(), dat->size(), insert);
    }
    using GmAndName = std::pair<SkQP::GMFactory, std::string>;
    std::vector<GmAndName> gmsWithNames;
    for (skiagm::GMFactory f : skiagm::GMRegistry::Range()) {
        std::string name = SkQP::GetGMName(f);
        if ((gmThresholds->empty() || gmThresholds->count(name) > 0)) {
            gmsWithNames.push_back(std::make_pair(f, std::move(name)));
        }
    }
    std::sort(gmsWithNames.begin(), gmsWithNames.end(),
              [](GmAndName u, GmAndName v) { return u.second < v.second; });
    gmlist->reserve(gmsWithNames.size());
    for (const GmAndName& gmn : gmsWithNames) {
        gmlist->push_back(gmn.first);
    }
}

static std::unique_ptr<sk_gpu_test::TestContext> make_test_context(SkQP::SkiaBackend backend) {
    using U = std::unique_ptr<sk_gpu_test::TestContext>;
    switch (backend) {
// TODO(halcanary): Fuchsia will have SK_SUPPORT_GPU and SK_VULKAN, but *not* SK_GL.
#ifdef SK_GL
        case SkQP::SkiaBackend::kGL:
            return U(sk_gpu_test::CreatePlatformGLTestContext(kGL_GrGLStandard, nullptr));
        case SkQP::SkiaBackend::kGLES:
            return U(sk_gpu_test::CreatePlatformGLTestContext(kGLES_GrGLStandard, nullptr));
#endif
#ifdef SK_VULKAN
        case SkQP::SkiaBackend::kVulkan:
            return U(sk_gpu_test::CreatePlatformVkTestContext(nullptr));
#endif
        default:
            return nullptr;
    }
}

static GrContextOptions context_options(skiagm::GM* gm = nullptr) {
    GrContextOptions grContextOptions;
    grContextOptions.fAllowPathMaskCaching = true;
    grContextOptions.fDisableDriverCorrectnessWorkarounds = true;
    if (gm) {
        gm->modifyGrContextOptions(&grContextOptions);
    }
    return grContextOptions;
}

static std::vector<SkQP::SkiaBackend> get_backends() {
    std::vector<SkQP::SkiaBackend> result;
    SkQP::SkiaBackend backends[] = {
        #ifdef SK_GL
        #ifndef SK_BUILD_FOR_ANDROID
        SkQP::SkiaBackend::kGL,  // Used for testing on desktop machines.
        #endif
        SkQP::SkiaBackend::kGLES,
        #endif  // SK_GL
        #ifdef SK_VULKAN
        SkQP::SkiaBackend::kVulkan,
        #endif
    };
    for (SkQP::SkiaBackend backend : backends) {
        std::unique_ptr<sk_gpu_test::TestContext> testCtx = make_test_context(backend);
        if (testCtx) {
            testCtx->makeCurrent();
            if (nullptr != testCtx->makeGrContext(context_options())) {
                result.push_back(backend);
            }
        }
    }
    SkASSERT_RELEASE(result.size() > 0);
    return result;
}

static void print_backend_info(const char* dstPath,
                               const std::vector<SkQP::SkiaBackend>& backends) {
#ifdef SK_ENABLE_DUMP_GPU
    SkFILEWStream out(dstPath);
    out.writeText("[\n");
    for (SkQP::SkiaBackend backend : backends) {
        if (std::unique_ptr<sk_gpu_test::TestContext> testCtx = make_test_context(backend)) {
            testCtx->makeCurrent();
            if (sk_sp<GrContext> ctx = testCtx->makeGrContext(context_options())) {
                SkString info = ctx->dump();
                // remove null
                out.write(info.c_str(), info.size());
                out.writeText(",\n");
            }
        }
    }
    out.writeText("]\n");
#endif
}

static void encode_png(const SkBitmap& src, const std::string& dst) {
    SkFILEWStream wStream(dst.c_str());
    SkPngEncoder::Options options;
    bool success = wStream.isValid() && SkPngEncoder::Encode(&wStream, src.pixmap(), options);
    SkASSERT_RELEASE(success);
}

static void write_to_file(const sk_sp<SkData>& src, const std::string& dst) {
    SkFILEWStream wStream(dst.c_str());
    bool success = wStream.isValid() && wStream.write(src->data(), src->size());
    SkASSERT_RELEASE(success);
}

////////////////////////////////////////////////////////////////////////////////

const char* SkQP::GetBackendName(SkQP::SkiaBackend b) {
    switch (b) {
        case SkQP::SkiaBackend::kGL:     return "gl";
        case SkQP::SkiaBackend::kGLES:   return "gles";
        case SkQP::SkiaBackend::kVulkan: return "vk";
    }
    return "";
}

std::string SkQP::GetGMName(SkQP::GMFactory f) {
    std::unique_ptr<skiagm::GM> gm(f ? f() : nullptr);
    return std::string(gm ? gm->getName() : "");
}

const char* SkQP::GetUnitTestName(SkQP::UnitTest t) { return t->name; }

SkQP::SkQP() {}

SkQP::~SkQP() {}

void SkQP::init(SkQPAssetManager* am, const char* reportDirectory) {
    SkASSERT_RELEASE(!fAssetManager);
    SkASSERT_RELEASE(am);
    fAssetManager = am;
    fReportDirectory = reportDirectory;

    SkGraphics::Init();
    gSkFontMgr_DefaultFactory = &ToolUtils::MakePortableFontMgr;

    /* If the file "skqp/rendertests.txt" does not exist or is empty, run all
       render tests.  Otherwise only run tests mentioned in that file.  */
    get_render_tests(fAssetManager, &fGMs, &fGMThresholds);
    /* If the file "skqp/unittests.txt" does not exist or is empty, run all gpu
       unit tests.  Otherwise only run tests mentioned in that file.  */
    get_unit_tests(fAssetManager, &fUnitTests);
    fSupportedBackends = get_backends();

    print_backend_info((fReportDirectory + "/grdump.txt").c_str(), fSupportedBackends);
}

std::tuple<SkQP::RenderOutcome, std::string> SkQP::evaluateGM(SkQP::SkiaBackend backend,
                                                              SkQP::GMFactory gmFact) {
    SkASSERT_RELEASE(fAssetManager);
    static constexpr SkQP::RenderOutcome kError = {INT_MAX, INT_MAX, INT64_MAX};
    static constexpr SkQP::RenderOutcome kPass = {0, 0, 0};

    std::unique_ptr<sk_gpu_test::TestContext> testCtx = make_test_context(backend);
    if (!testCtx) {
        return std::make_tuple(kError, "Skia Failure: test context");
    }
    testCtx->makeCurrent();

    SkASSERT(gmFact);
    std::unique_ptr<skiagm::GM> gm(gmFact());
    SkASSERT(gm);
    const char* const name = gm->getName();
    const SkISize size = gm->getISize();
    const int w = size.width();
    const int h = size.height();
    const SkImageInfo info =
        SkImageInfo::Make(w, h, skqp::kColorType, kPremul_SkAlphaType, nullptr);
    const SkSurfaceProps props(0, SkSurfaceProps::kLegacyFontHost_InitType);

    sk_sp<SkSurface> surf = SkSurface::MakeRenderTarget(
            testCtx->makeGrContext(context_options(gm.get())).get(),
            SkBudgeted::kNo, info, 0, &props);
    if (!surf) {
        return std::make_tuple(kError, "Skia Failure: gr-context");
    }
    gm->draw(surf->getCanvas());

    SkBitmap image;
    image.allocPixels(SkImageInfo::Make(w, h, skqp::kColorType, skqp::kAlphaType));

    // SkColorTypeBytesPerPixel should be constexpr, but is not.
    SkASSERT(SkColorTypeBytesPerPixel(skqp::kColorType) == sizeof(uint32_t));
    // Call readPixels because we need to compare pixels.
    if (!surf->readPixels(image.pixmap(), 0, 0)) {
        return std::make_tuple(kError, "Skia Failure: read pixels");
    }
    int64_t passingThreshold = fGMThresholds.empty() ? -1 : fGMThresholds[std::string(name)];

    if (-1 == passingThreshold) {
        return std::make_tuple(kPass, "");
    }
    skqp::ModelResult modelResult =
        skqp::CheckAgainstModel(name, image.pixmap(), fAssetManager);

    if (!modelResult.fErrorString.empty()) {
        return std::make_tuple(kError, std::move(modelResult.fErrorString));
    }
    fRenderResults.push_back(SkQP::RenderResult{backend, gmFact, modelResult.fOutcome});
    if (modelResult.fOutcome.fMaxError <= passingThreshold) {
        return std::make_tuple(kPass, "");
    }
    std::string imagesDirectory = fReportDirectory + "/" IMAGES_DIRECTORY_PATH;
    if (!sk_mkdir(imagesDirectory.c_str())) {
        SkDebugf("ERROR: sk_mkdir('%s');\n", imagesDirectory.c_str());
        return std::make_tuple(modelResult.fOutcome, "");
    }
    std::ostringstream tmp;
    tmp << imagesDirectory << '/' << SkQP::GetBackendName(backend) << '_' << name << '_';
    std::string imagesPathPrefix1 = tmp.str();
    tmp = std::ostringstream();
    tmp << imagesDirectory << '/' << PATH_MODEL << '_' << name << '_';
    std::string imagesPathPrefix2 = tmp.str();
    encode_png(image,                  imagesPathPrefix1 + PATH_IMG_PNG);
    encode_png(modelResult.fErrors,    imagesPathPrefix1 + PATH_ERR_PNG);
    write_to_file(modelResult.fMaxPng, imagesPathPrefix2 + PATH_MAX_PNG);
    write_to_file(modelResult.fMinPng, imagesPathPrefix2 + PATH_MIN_PNG);
    return std::make_tuple(modelResult.fOutcome, "");
}

std::vector<std::string> SkQP::executeTest(SkQP::UnitTest test) {
    SkASSERT_RELEASE(fAssetManager);
    struct : public skiatest::Reporter {
        std::vector<std::string> fErrors;
        void reportFailed(const skiatest::Failure& failure) override {
            SkString desc = failure.toString();
            fErrors.push_back(std::string(desc.c_str(), desc.size()));
        }
    } r;
    GrContextOptions options;
    options.fDisableDriverCorrectnessWorkarounds = true;
    if (test->fContextOptionsProc) {
        test->fContextOptionsProc(&options);
    }
    test->proc(&r, options);
    fUnitTestResults.push_back(UnitTestResult{test, r.fErrors});
    return r.fErrors;
}

////////////////////////////////////////////////////////////////////////////////

static constexpr char kDocHead[] =
    "<!doctype html>\n"
    "<html lang=\"en\">\n"
    "<head>\n"
    "<meta charset=\"UTF-8\">\n"
    "<title>SkQP Report</title>\n"
    "<style>\n"
    "img { max-width:48%; border:1px green solid;\n"
    "      image-rendering: pixelated;\n"
    "      background-image:url('data:image/png;base64,iVBORw0KGgoA"
    "AAANSUhEUgAAABAAAAAQCAAAAAA6mKC9AAAAAXNSR0IArs4c6QAAAAJiS0dEAP+H"
    "j8y/AAAACXBIWXMAAA7DAAAOwwHHb6hkAAAAB3RJTUUH3gUBEi4DGRAQYgAAAB1J"
    "REFUGNNjfMoAAVJQmokBDdBHgPE/lPFsYN0BABdaAwN6tehMAAAAAElFTkSuQmCC"
    "'); }\n"
    "</style>\n"
    "<script>\n"
    "function ce(t) { return document.createElement(t); }\n"
    "function ct(n) { return document.createTextNode(n); }\n"
    "function ac(u,v) { return u.appendChild(v); }\n"
    "function br(u) { ac(u, ce(\"br\")); }\n"
    "function ma(s, c) { var a = ce(\"a\"); a.href = s; ac(a, c); return a; }\n"
    "function f(backend, gm, e1, e2, e3) {\n"
    "  var b = ce(\"div\");\n"
    "  var x = ce(\"h2\");\n"
    "  var t = backend + \"_\" + gm;\n"
    "  ac(x, ct(t));\n"
    "  ac(b, x);\n"
    "  ac(b, ct(\"backend: \" + backend));\n"
    "  br(b);\n"
    "  ac(b, ct(\"gm name: \" + gm));\n"
    "  br(b);\n"
    "  ac(b, ct(\"maximum error: \" + e1));\n"
    "  br(b);\n"
    "  ac(b, ct(\"bad pixel counts: \" + e2));\n"
    "  br(b);\n"
    "  ac(b, ct(\"total error: \" + e3));\n"
    "  br(b);\n"
    "  var q = \"" IMAGES_DIRECTORY_PATH "/\" + backend + \"_\" + gm + \"_\";\n"
    "  var p = \"" IMAGES_DIRECTORY_PATH "/"   PATH_MODEL  "_\" + gm + \"_\";\n"
    "  var i = ce(\"img\");\n"
    "  i.src = q + \"" PATH_IMG_PNG "\";\n"
    "  i.alt = \"img\";\n"
    "  ac(b, ma(i.src, i));\n"
    "  i = ce(\"img\");\n"
    "  i.src = q + \"" PATH_ERR_PNG "\";\n"
    "  i.alt = \"err\";\n"
    "  ac(b, ma(i.src, i));\n"
    "  br(b);\n"
    "  ac(b, ct(\"Expectation: \"));\n"
    "  ac(b, ma(p + \"" PATH_MAX_PNG "\", ct(\"max\")));\n"
    "  ac(b, ct(\" | \"));\n"
    "  ac(b, ma(p + \"" PATH_MIN_PNG "\", ct(\"min\")));\n"
    "  ac(b, ce(\"hr\"));\n"
    "  b.id = backend + \":\" + gm;\n"
    "  ac(document.body, b);\n"
    "  l = ce(\"li\");\n"
    "  ac(l, ct(\"[\" + e3 + \"] \"));\n"
    "  ac(l, ma(\"#\" + backend +\":\"+ gm , ct(t)));\n"
    "  ac(document.getElementById(\"toc\"), l);\n"
    "}\n"
    "function main() {\n";

static constexpr char kDocMiddle[] =
    "}\n"
    "</script>\n"
    "</head>\n"
    "<body onload=\"main()\">\n"
    "<h1>SkQP Report</h1>\n";

static constexpr char kDocTail[] =
    "<ul id=\"toc\"></ul>\n"
    "<hr>\n"
    "<p>Left image: test result<br>\n"
    "Right image: errors (white = no error, black = smallest error, red = biggest error; "
    "other errors are a color between black and red.)</p>\n"
    "<hr>\n"
    "</body>\n"
    "</html>\n";

template <typename T>
inline void write(SkWStream* wStream, const T& text) {
    wStream->write(text.c_str(), text.size());
}

void SkQP::makeReport() {
    SkASSERT_RELEASE(fAssetManager);
    int glesErrorCount = 0, vkErrorCount = 0, gles = 0, vk = 0;

    if (!sk_isdir(fReportDirectory.c_str())) {
        SkDebugf("Report destination does not exist: '%s'\n", fReportDirectory.c_str());
        return;
    }
    SkFILEWStream csvOut(SkOSPath::Join(fReportDirectory.c_str(), kRenderTestCSVReport).c_str());
    SkFILEWStream htmOut(SkOSPath::Join(fReportDirectory.c_str(), kRenderTestReportPath).c_str());
    SkASSERT_RELEASE(csvOut.isValid() && htmOut.isValid());
    htmOut.writeText(kDocHead);
    for (const SkQP::RenderResult& run : fRenderResults) {
        switch (run.fBackend) {
            case SkQP::SkiaBackend::kGLES: ++gles; break;
            case SkQP::SkiaBackend::kVulkan: ++vk; break;
            default: break;
        }
        const char* backendName = SkQP::GetBackendName(run.fBackend);
        std::string gmName = SkQP::GetGMName(run.fGM);
        const SkQP::RenderOutcome& outcome = run.fOutcome;
        auto str = SkStringPrintf("\"%s\",\"%s\",%d,%d,%" PRId64, backendName, gmName.c_str(),
                                  outcome.fMaxError, outcome.fBadPixelCount, outcome.fTotalError);
        write(&csvOut, SkStringPrintf("%s\n", str.c_str()));

        int64_t passingThreshold = fGMThresholds.empty() ? 0 : fGMThresholds[gmName];
        if (passingThreshold == -1 || outcome.fMaxError <= passingThreshold) {
            continue;
        }
        write(&htmOut, SkStringPrintf("  f(%s);\n", str.c_str()));
        switch (run.fBackend) {
            case SkQP::SkiaBackend::kGLES: ++glesErrorCount; break;
            case SkQP::SkiaBackend::kVulkan: ++vkErrorCount; break;
            default: break;
        }
    }
    htmOut.writeText(kDocMiddle);
    write(&htmOut, SkStringPrintf("<p>gles errors: %d (of %d)</br>\n"
                                  "vk errors: %d (of %d)</p>\n",
                                  glesErrorCount, gles, vkErrorCount, vk));
    htmOut.writeText(kDocTail);
    SkFILEWStream unitOut(SkOSPath::Join(fReportDirectory.c_str(), kUnitTestReportPath).c_str());
    SkASSERT_RELEASE(unitOut.isValid());
    for (const SkQP::UnitTestResult& result : fUnitTestResults) {
        unitOut.writeText(GetUnitTestName(result.fUnitTest));
        if (result.fErrors.empty()) {
            unitOut.writeText(" PASSED\n* * *\n");
        } else {
            write(&unitOut, SkStringPrintf(" FAILED (%u errors)\n", result.fErrors.size()));
            for (const std::string& err : result.fErrors) {
                write(&unitOut, err);
                unitOut.newline();
            }
            unitOut.writeText("* * *\n");
        }
    }
}