GPU-CTS Program

Add new application, called GPU-CTS (GPU Compatibility Test Suite),
which executes skia gms against OpenGL and Vulkan backends.  Makes use
of googletest library for consistancy with Android CTS programs.

Add googletest to DEPS

gm_knowledge.h header as a stub for future work on validating gm output.

gm_runner can be re-used in other programs.  Talks to Skia and GM with a
simple API.

gpuctx executable wraps gm_runner and googletest together.

Change-Id: Ie7350b22164fa73e44121c39b0f36da4038a700b
Reviewed-on: https://skia-review.googlesource.com/56601
Reviewed-by: Hal Canary <halcanary@google.com>
Commit-Queue: Hal Canary <halcanary@google.com>

8 files changed, 474 insertions, 0 deletions

diff --git a/BUILD.gn b/BUILD.gn
index 33922d708d..14ce04a298 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -1634,6 +1634,24 @@ if (skia_enable_tools) {
     ]
   }
 
+  if (!is_win) {
+    test_app("gpucts") {
+      sources = [
+        "dm/DMGpuTestProcs.cpp",
+        "tools/gpucts/gm_knowledge.c",
+        "tools/gpucts/gm_runner.cpp",
+        "tools/gpucts/gpucts.cpp",
+      ]
+      deps = [
+        ":gm",
+        ":gpu_tool_utils",
+        ":skia",
+        ":tests",
+        "//third_party/googletest",
+      ]
+    }
+  }
+
   if (skia_enable_gpu) {
     test_app("viewer") {
       is_shared_library = is_android
diff --git a/DEPS b/DEPS
index 7eb63b3977..77453c1a2a 100644
--- a/DEPS
+++ b/DEPS
@@ -7,6 +7,7 @@ deps = {
   "third_party/externals/dng_sdk"       : "https://android.googlesource.com/platform/external/dng_sdk.git@96443b262250c390b0caefbf3eed8463ba35ecae",
   "third_party/externals/expat"         : "https://android.googlesource.com/platform/external/expat.git@android-6.0.1_r55",
   "third_party/externals/freetype"      : "https://skia.googlesource.com/third_party/freetype2.git@447a0b62634802d8acdb56008cff5ff4e50be244",
+  "third_party/externals/googletest"    : "https://android.googlesource.com/platform/external/googletest@dd43b9998e9a44a579a7aba6c1309407d1a5ed95",
   "third_party/externals/harfbuzz"      : "https://skia.googlesource.com/third_party/harfbuzz.git@1.4.2",
   "third_party/externals/icu"           : "https://chromium.googlesource.com/chromium/deps/icu.git@ec9c1133693148470ffe2e5e53576998e3650c1d",
   "third_party/externals/imgui"         : "https://github.com/ocornut/imgui.git@6384eee34f08cb7eab8d835043e1738e4adcdf75",
diff --git a/third_party/googletest/BUILD.gn b/third_party/googletest/BUILD.gn
new file mode 100644
index 0000000000..518360bdbe
--- /dev/null
+++ b/third_party/googletest/BUILD.gn
@@ -0,0 +1,16 @@
+# Copyright 2017 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("../third_party.gni")
+
+if (!is_win) {
+  third_party("googletest") {
+    public_include_dirs = [ "../externals/googletest/googletest/include" ]
+    include_dirs = [ "../externals/googletest/googletest" ]
+    sources = [
+      "../externals/googletest/googletest/src/gtest-all.cc",
+    ]
+  }
+}
diff --git a/tools/gpucts/gm_knowledge.c b/tools/gpucts/gm_knowledge.c
new file mode 100644
index 0000000000..8dbdd00f09
--- /dev/null
+++ b/tools/gpucts/gm_knowledge.c
@@ -0,0 +1,12 @@
+/*
+ * Copyright 2017 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include "gm_knowledge.h"
+
+// placeholder function definitions:
+float GMK_Check(GMK_ImageData d, const char* n) { return 0; }
+bool GMK_IsGoodGM(const char* n) { return true; }
diff --git a/tools/gpucts/gm_knowledge.h b/tools/gpucts/gm_knowledge.h
new file mode 100644
index 0000000000..d9d71e05c5
--- /dev/null
+++ b/tools/gpucts/gm_knowledge.h
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2017 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+#ifndef gm_knowledge_DEFINED
+#define gm_knowledge_DEFINED
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stdbool.h>
+
+/**
+A structure representing an image.  pix should either be nullptr (representing
+a missing image) or point to a block of memory width*height in size.
+
+Each pixel is an un-pre-multiplied RGBA color:
+    void set_color(GMK_ImageData* data, int x, int y,
+                   unsigned char r, unsigned char g, unsigned char b, unsigned char a) {
+        data->pix[x + data->width * y] = (r << 0) | (g << 8) | (b << 16) | (a << 24);
+    }
+ */
+typedef struct {
+    const uint32_t* pix;
+    int width;
+    int height;
+} GMK_ImageData;
+
+/**
+Check if the given test image matches the expected results.
+
+@param data     the image
+@param gm_name  the name of the rendering test that produced the image
+
+@return 0 if the test passes, otherwise a positive number representing how
+         badly it failed.
+ */
+float GMK_Check(GMK_ImageData data, const char* gm_name);
+
+/**
+Check to see if the given test has expected results.
+
+@param gm_name  the name of a rendering test.
+
+@return true of expected results are known for the given test.
+*/
+bool GMK_IsGoodGM(const char* gm_name);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // gm_knowledge_DEFINED
diff --git a/tools/gpucts/gm_runner.cpp b/tools/gpucts/gm_runner.cpp
new file mode 100644
index 0000000000..92d918106d
--- /dev/null
+++ b/tools/gpucts/gm_runner.cpp
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2017 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include "gm_runner.h"
+
+#include <algorithm>
+
+#include "SkGraphics.h"
+#include "SkSurface.h"
+#include "gm.h"
+
+#if SK_SUPPORT_GPU
+
+#include "GrContextFactory.h"
+
+using sk_gpu_test::GrContextFactory;
+
+namespace gm_runner {
+
+static GrContextFactory::ContextType to_context_type(SkiaBackend backend) {
+    switch (backend) {
+        case SkiaBackend::kGL:     return GrContextFactory::kGL_ContextType;
+        case SkiaBackend::kGLES:   return GrContextFactory::kGLES_ContextType;
+        case SkiaBackend::kVulkan: return GrContextFactory::kVulkan_ContextType;
+    }
+    SkDEBUGFAIL(""); return (GrContextFactory::ContextType)0;
+}
+
+const char* GetBackendName(SkiaBackend backend) {
+    return GrContextFactory::ContextTypeName(to_context_type(backend));
+}
+
+bool BackendSupported(SkiaBackend backend, GrContextFactory* contextFactory) {
+    return contextFactory->get(to_context_type(backend)) != nullptr;
+}
+
+
+GMK_ImageData Evaluate(SkiaBackend backend,
+                       GMFactory gmFact,
+                       GrContextFactory* contextFactory,
+                       std::vector<uint32_t>* storage) {
+    SkASSERT(contextFactory);
+    SkASSERT(gmFact);
+    SkASSERT(storage);
+    std::unique_ptr<skiagm::GM> gm(gmFact(nullptr));
+    SkASSERT(gm.get());
+    int w = SkScalarRoundToInt(gm->width());
+    int h = SkScalarRoundToInt(gm->height());
+    GrContext* context = contextFactory->get(to_context_type(backend));
+    if (!context) {
+        return GMK_ImageData{nullptr, w, h};
+    }
+    SkASSERT(context);
+    constexpr SkColorType ct = kRGBA_8888_SkColorType;
+
+    sk_sp<SkSurface> s = SkSurface::MakeRenderTarget(
+            context, SkBudgeted::kNo, SkImageInfo::Make(w, h, ct, kPremul_SkAlphaType));
+    if (!s) {
+        return GMK_ImageData{nullptr, w, h};
+    }
+    gm->draw(s->getCanvas());
+
+    storage->resize(w * h);
+    uint32_t* pix = storage->data();
+    SkASSERT(SkColorTypeBytesPerPixel(ct) == sizeof(uint32_t));
+    SkAssertResult(s->readPixels(SkImageInfo::Make(w, h, ct, kUnpremul_SkAlphaType),
+                                 pix, w * sizeof(uint32_t), 0, 0));
+    return GMK_ImageData{pix, w, h};
+}
+
+std::unique_ptr<GrContextFactory> make_gr_context_factory() {
+    GrContextOptions grContextOptions; // TODO: change options?
+    return std::unique_ptr<GrContextFactory>(new GrContextFactory(grContextOptions));
+}
+
+SkiaContext::SkiaContext() : fGrContextFactory(make_gr_context_factory()) {
+    SkGraphics::Init();
+}
+
+void SkiaContext::resetContextFactory() {
+    fGrContextFactory->destroyContexts();
+}
+
+SkiaContext::~SkiaContext() {}
+
+}  // namespace gm_runner
+
+#else
+namespace sk_gpu_test {
+    class GrContextFactory {};
+}
+namespace gm_runner {
+SkiaContext::SkiaContext() {}
+SkiaContext::~SkiaContext() {}
+void SkiaContext::resetContextFactory() {}
+bool BackendSupported(SkiaBackend, sk_gpu_test::GrContextFactory*) { return false; }
+GMK_ImageData Evaluate(SkiaBackend, GMFactory,
+                       sk_gpu_test::GrContextFactory*, std::vector<uint32_t>*) {
+    return GMK_ImageData{nullptr, 0, 0};
+}
+const char* GetBackendName(SkiaBackend backend) { return "Unknown"; }
+}  // namespace gm_runner
+#endif
+
+namespace gm_runner {
+
+std::vector<GMFactory> GetGMFactories() {
+    std::vector<GMFactory> result;
+    for (const skiagm::GMRegistry* r = skiagm::GMRegistry::Head(); r; r = r->next()) {
+        result.push_back(r->factory());
+    }
+    struct {
+        bool operator()(GMFactory u, GMFactory v) const { return GetGMName(u) < GetGMName(v); }
+    } less;
+    std::sort(result.begin(), result.end(), less);
+    return result;
+}
+
+std::string GetGMName(GMFactory gmFactory) {
+    SkASSERT(gmFactory);
+    std::unique_ptr<skiagm::GM> gm(gmFactory(nullptr));
+    SkASSERT(gm);
+    return std::string(gm->getName());
+}
+}  // namespace gm_runner
diff --git a/tools/gpucts/gm_runner.h b/tools/gpucts/gm_runner.h
new file mode 100644
index 0000000000..6324914799
--- /dev/null
+++ b/tools/gpucts/gm_runner.h
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2017 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+#ifndef gm_runner_DEFINED
+#define gm_runner_DEFINED
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gm_knowledge.h"
+
+/**
+A Skia GM is a single rendering test that can be executed on any Skia backend Canvas.
+*/
+namespace skiagm {
+   class GM;
+}
+
+namespace sk_gpu_test {
+    class GrContextFactory;
+}
+
+namespace gm_runner {
+
+using GMFactory = skiagm::GM* (*)(void*);
+
+enum class SkiaBackend {
+    kGL,
+    kGLES,
+    kVulkan,
+};
+
+/**
+This class initializes Skia and a GrContextFactory.
+*/
+struct SkiaContext {
+    SkiaContext();
+    ~SkiaContext();
+    void resetContextFactory();
+    std::unique_ptr<sk_gpu_test::GrContextFactory> fGrContextFactory;
+};
+
+bool BackendSupported(SkiaBackend, sk_gpu_test::GrContextFactory*);
+
+/**
+@return a list of all Skia GMs in lexicographic order.
+*/
+std::vector<GMFactory> GetGMFactories();
+
+/**
+@return a descriptive name for the GM.
+*/
+std::string GetGMName(GMFactory);
+/**
+@return a descriptive name for the backend.
+*/
+const char* GetBackendName(SkiaBackend);
+
+/**
+Execute the given GM on the given Skia backend.  Then copy the pixels into the
+storage (overwriting existing contents of storage).
+
+@return the rendered image.  Return a null ImageData on error.
+*/
+GMK_ImageData Evaluate(SkiaBackend,
+                       GMFactory,
+                       sk_gpu_test::GrContextFactory*,
+                       std::vector<uint32_t>* storage);
+
+}  // namespace gm_runner
+
+#endif  // gm_runner_DEFINED
diff --git a/tools/gpucts/gpucts.cpp b/tools/gpucts/gpucts.cpp
new file mode 100644
index 0000000000..d244d91e2d
--- /dev/null
+++ b/tools/gpucts/gpucts.cpp
@@ -0,0 +1,165 @@
+/*
+ * Copyright 2017 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include "gm_runner.h"
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wused-but-marked-unused"
+#endif
+
+#include "gtest/gtest.h"
+
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
+#include "Test.h"
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct GMTestCase {
+    gm_runner::GMFactory fGMFactory;
+    gm_runner::SkiaBackend fBackend;
+    sk_gpu_test::GrContextFactory* fGrContextFactory;
+};
+
+struct GMTest : public testing::Test {
+    GMTestCase fTest;
+    GMTest(GMTestCase t) : fTest(t) {}
+    void TestBody() override {
+        if (!fTest.fGMFactory) {
+            EXPECT_TRUE(gm_runner::BackendSupported(fTest.fBackend, fTest.fGrContextFactory));
+            return;
+        }
+        std::vector<uint32_t> pixels;
+        GMK_ImageData imgData = gm_runner::Evaluate(
+                fTest.fBackend, fTest.fGMFactory, fTest.fGrContextFactory, &pixels);
+        EXPECT_TRUE(imgData.pix);
+        if (!imgData.pix) {
+            return;
+        }
+        std::string gmName = gm_runner::GetGMName(fTest.fGMFactory);
+        float result = GMK_Check(imgData, gmName.c_str());
+        EXPECT_EQ(result, 0);
+    }
+};
+
+struct GMTestFactory : public testing::internal::TestFactoryBase {
+    GMTestCase fTest;
+    GMTestFactory(GMTestCase t) : fTest(t) {}
+    testing::Test* CreateTest() override { return new GMTest(fTest); }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct UnitTestData {
+    gm_runner::SkiaContext* fContext;
+    skiatest::TestProc fProc;
+};
+
+struct UnitTest : public testing::Test {
+    UnitTestData fUnitTestData;
+    UnitTest(UnitTestData d) : fUnitTestData(d) {}
+    void TestBody() override {
+        struct : skiatest::Reporter {
+            void reportFailed(const skiatest::Failure& failure) override {
+                SkString desc = failure.toString();
+                SK_ABORT("");
+                GTEST_NONFATAL_FAILURE_(desc.c_str());
+            }
+        } r;
+        fUnitTestData.fContext->resetContextFactory();
+        fUnitTestData.fProc(&r, fUnitTestData.fContext->fGrContextFactory.get());
+    }
+};
+
+struct UnitTestFactory : testing::internal::TestFactoryBase {
+    UnitTestData fUnitTestData;
+    UnitTestFactory(UnitTestData d) : fUnitTestData(d) {}
+    testing::Test* CreateTest() override { return new UnitTest(fUnitTestData); }
+};
+
+std::vector<const skiatest::Test*> GetUnitTests() {
+    // Unit Tests
+    std::vector<const skiatest::Test*> tests;
+    for (const skiatest::TestRegistry* r = skiatest::TestRegistry::Head(); r; r = r->next()) {
+        tests.push_back(&r->factory());
+    }
+    struct {
+        bool operator()(const skiatest::Test* u, const skiatest::Test* v) const {
+            return strcmp(u->name, v->name) < 0;
+        }
+    } less;
+    std::sort(tests.begin(), tests.end(), less);
+    return tests;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+static void reg_test(const char* test, const char* testCase,
+                     testing::internal::TestFactoryBase* fact) {
+    testing::internal::MakeAndRegisterTestInfo(
+                        test,
+                        testCase,
+                        nullptr,
+                        nullptr,
+                        testing::internal::CodeLocation(__FILE__, __LINE__),
+                        testing::internal::GetTestTypeId(),
+                        testing::Test::SetUpTestCase,
+                        testing::Test::TearDownTestCase,
+                        fact);
+}
+
+int main(int argc, char** argv) {
+    testing::InitGoogleTest(&argc, argv);
+    gm_runner::SkiaContext context;
+    sk_gpu_test::GrContextFactory* grContextFactory = context.fGrContextFactory.get();
+
+    // Rendering Tests
+    gm_runner::SkiaBackend backends[] = {
+        #ifndef SK_BUILD_FOR_ANDROID
+        gm_runner::SkiaBackend::kGL,  // Used for testing on desktop machines.
+        #endif
+        gm_runner::SkiaBackend::kGLES,
+        gm_runner::SkiaBackend::kVulkan,
+    };
+    std::vector<gm_runner::GMFactory> gms = gm_runner::GetGMFactories();
+    for (auto backend : backends) {
+        const char* backendName = GetBackendName(backend);
+        std::string test = std::string("SkiaGM_") + backendName;
+        reg_test(test.c_str(), "BackendSupported",
+                 new GMTestFactory(GMTestCase{nullptr, backend, grContextFactory}));
+
+        if (!gm_runner::BackendSupported(backend, context.fGrContextFactory.get())) {
+            continue;
+        }
+        for (auto gmFactory : gms) {
+            std::string gmName = gm_runner::GetGMName(gmFactory);
+            if (!GMK_IsGoodGM(gmName.c_str())) {
+                continue;
+            }
+            #ifdef SK_DEBUG
+                // The following test asserts on my phone.
+                // TODO(halcanary):  fix this.
+                if(gmName == std::string("complexclip3_simple") &&
+                   backend == gm_runner::SkiaBackend::kGLES) {
+                    continue;
+                }
+            #endif
+            reg_test(test.c_str(), gmName.c_str(),
+                     new GMTestFactory(GMTestCase{gmFactory, backend, grContextFactory}));
+      }
+    }
+
+    for (const skiatest::Test* test : GetUnitTests()) {
+            reg_test("Skia_Unit_Tests", test->name,
+                new UnitTestFactory(UnitTestData{&context, test->proc}));
+    }
+    return RUN_ALL_TESTS();
+}
+