Revert of New version of the bbh shootout (https://codereview.chromium.org/186973005/)

Reason for revert:
breaking build on at least 12 configurations.

Original issue's description:
> New version of the bbh shootout.
> There were a bunch of problems with the existing code, all of which I would of ignored, except it could only benchmark none and rtree.
> The new version measures the same numbers, in almost exactly the same way, it's the structure over the top of the actual test runner that I changed.
> Now it only loads the pictures once, it is configurable in which bbh's it runs and how may loops on them it does, it uses standard command line processing, and it does not do so much redundant work so it runs quicker. It is also much easier to understand and change IMHO.
>
> Doing this so I can reasonably compare the new QuadTree implementation.
> BUG=skia:2242
>
> Committed: http://code.google.com/p/skia/source/detail?r=13736

R=tomhudson@google.com, mtklein@google.com, reed@google.com, iancottrell@google.com
TBR=iancottrell@google.com, mtklein@google.com, reed@google.com, tomhudson@google.com
NOTREECHECKS=true
NOTRY=true
BUG=skia:2242

Author: halcanary@google.com

Review URL: https://codereview.chromium.org/194903003

git-svn-id: http://skia.googlecode.com/svn/trunk@13737 2bbb7eff-a529-9590-31e7-b0007b416f81

1 files changed, 319 insertions, 106 deletions

diff --git a/tools/bbh_shootout.cpp b/tools/bbh_shootout.cpp
index 6475ab18b4..fa1739c3b5 100644
--- a/tools/bbh_shootout.cpp
+++ b/tools/bbh_shootout.cpp
@@ -15,30 +15,155 @@
 #include "SkStream.h"
 #include "SkString.h"
 #include "SkTArray.h"
-#include "SkCommandLineFlags.h"
+#include "TimerData.h"
 
-typedef sk_tools::PictureRenderer::BBoxHierarchyType BBoxType;
-static const int kBBoxTypeCount = sk_tools::PictureRenderer::kLast_BBoxHierarchyType + 1;
+static const int kNumNormalRecordings = 10;
+static const int kNumRTreeRecordings = 10;
+static const int kNumPlaybacks = 1;
+static const size_t kNumBaseBenchmarks = 3;
+static const size_t kNumTileSizes = 3;
+static const size_t kNumBbhPlaybackBenchmarks = 3;
+static const size_t kNumBenchmarks = kNumBaseBenchmarks + kNumBbhPlaybackBenchmarks;
+
+enum BenchmarkType {
+    kNormal_BenchmarkType = 0,
+    kRTree_BenchmarkType,
+};
+
+struct Histogram {
+    Histogram() {
+        // Make fCpuTime negative so that we don't mess with stats:
+        fCpuTime = SkIntToScalar(-1);
+    }
+    SkScalar fCpuTime;
+    SkString fPath;
+};
 
 
-DEFINE_string2(skps, r, "", "The list of SKPs to benchmark.");
-DEFINE_string(bbh, "", "The set of bbox types to test. If empty, all are tested. "
-                       "Should be one or more of none, quadtree, rtree, tilegrid.");
-DEFINE_int32(record, 100, "Number of times to record each SKP.");
-DEFINE_int32(playback, 1, "Number of times to playback each SKP.");
-DEFINE_int32(tilesize, 256, "The size of a tile.");
+////////////////////////////////////////////////////////////////////////////////
+// Defined below.
+struct BenchmarkControl;
 
-struct Measurement {
+typedef void (*BenchmarkFunction)
+    (const BenchmarkControl&, const SkString&, SkPicture*, BenchTimer*);
+
+static void benchmark_playback(
+    const BenchmarkControl&, const SkString&, SkPicture*, BenchTimer*);
+static void benchmark_recording(
+    const BenchmarkControl&, const SkString&, SkPicture*, BenchTimer*);
+////////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Acts as a POD containing information needed to run a benchmark.
+ * Provides static methods to poll benchmark info from an index.
+ */
+struct BenchmarkControl {
+    SkISize fTileSize;
+    BenchmarkType fType;
+    BenchmarkFunction fFunction;
     SkString fName;
-    SkScalar fRecordAverage[kBBoxTypeCount];
-    SkScalar fPlaybackAverage[kBBoxTypeCount];
+
+    /**
+     * Will construct a BenchmarkControl instance from an index between 0 an kNumBenchmarks.
+     */
+    static BenchmarkControl Make(size_t i) {
+        SkASSERT(kNumBenchmarks > i);
+        BenchmarkControl benchControl;
+        benchControl.fTileSize = GetTileSize(i);
+        benchControl.fType = GetBenchmarkType(i);
+        benchControl.fFunction = GetBenchmarkFunc(i);
+        benchControl.fName = GetBenchmarkName(i);
+        return benchControl;
+    }
+
+    enum BaseBenchmarks {
+        kNormalRecord = 0,
+        kRTreeRecord,
+        kNormalPlayback,
+    };
+
+    static SkISize fTileSizes[kNumTileSizes];
+
+    static SkISize GetTileSize(size_t i) {
+        // Two of the base benchmarks don't need a tile size. But to maintain simplicity
+        // down the pipeline we have to let a couple of values unused.
+        if (i < kNumBaseBenchmarks) {
+            return SkISize::Make(256, 256);
+        }
+        if (i >= kNumBaseBenchmarks && i < kNumBenchmarks) {
+            return fTileSizes[i - kNumBaseBenchmarks];
+        }
+        SkASSERT(0);
+        return SkISize::Make(0, 0);
+    }
+
+    static BenchmarkType GetBenchmarkType(size_t i) {
+        if (i < kNumBaseBenchmarks) {
+            switch (i) {
+            case kNormalRecord:
+                return kNormal_BenchmarkType;
+            case kNormalPlayback:
+                return kNormal_BenchmarkType;
+            case kRTreeRecord:
+                return kRTree_BenchmarkType;
+            }
+        }
+        if (i < kNumBenchmarks) {
+            return kRTree_BenchmarkType;
+        }
+        SkASSERT(0);
+        return kRTree_BenchmarkType;
+    }
+
+    static BenchmarkFunction GetBenchmarkFunc(size_t i) {
+        // Base functions.
+        switch (i) {
+            case kNormalRecord:
+                return benchmark_recording;
+            case kNormalPlayback:
+                return benchmark_playback;
+            case kRTreeRecord:
+                return benchmark_recording;
+        }
+        // RTree playbacks
+        if (i < kNumBenchmarks) {
+            return benchmark_playback;
+        }
+        SkASSERT(0);
+        return NULL;
+    }
+
+    static SkString GetBenchmarkName(size_t i) {
+        // Base benchmark names
+        switch (i) {
+            case kNormalRecord:
+                return SkString("normal_recording");
+            case kNormalPlayback:
+                return SkString("normal_playback");
+            case kRTreeRecord:
+                return SkString("rtree_recording");
+        }
+        // RTree benchmark names.
+        if (i < kNumBenchmarks) {
+            SkASSERT(i >= kNumBaseBenchmarks);
+            SkString name;
+            name.printf("rtree_playback_%dx%d",
+                    fTileSizes[i - kNumBaseBenchmarks].fWidth,
+                    fTileSizes[i - kNumBaseBenchmarks].fHeight);
+            return name;
+
+        } else {
+            SkASSERT(0);
+        }
+        return SkString("");
+    }
+
 };
 
-const char* kBBoxHierarchyTypeNames[kBBoxTypeCount] = {
-    "none", // kNone_BBoxHierarchyType
-    "quadtree", // kQuadTree_BBoxHierarchyType
-    "rtree", // kRTree_BBoxHierarchyType
-    "tilegrid", // kTileGrid_BBoxHierarchyType
+SkISize BenchmarkControl::fTileSizes[kNumTileSizes] = {
+    SkISize::Make(256, 256),
+    SkISize::Make(512, 512),
+    SkISize::Make(1024, 1024),
 };
 
 static SkPicture* pic_from_path(const char path[]) {
@@ -51,90 +176,162 @@ static SkPicture* pic_from_path(const char path[]) {
 }
 
 /**
+ * Returns true when a tiled renderer with no bounding box hierarchy produces the given bitmap.
+ */
+static bool compare_picture(const SkString& path, const SkBitmap& inBitmap, SkPicture* pic) {
+    SkBitmap* bitmap;
+    sk_tools::TiledPictureRenderer renderer;
+    renderer.setBBoxHierarchyType(sk_tools::PictureRenderer::kNone_BBoxHierarchyType);
+    renderer.init(pic);
+    renderer.setup();
+    renderer.render(&path, &bitmap);
+    SkAutoTDelete<SkBitmap> bmDeleter(bitmap);
+    renderer.end();
+
+    if (bitmap->getSize() != inBitmap.getSize()) {
+        return false;
+    }
+    return !memcmp(bitmap->getPixels(), inBitmap.getPixels(), bitmap->getSize());
+}
+
+/**
  * This function is the sink to which all work ends up going.
- * @param renderer The renderer to use to perform the work.
- *                 To measure rendering, use a TiledPictureRenderer.
- *                 To measure recording, use a RecordPictureRenderer.
- * @param bBoxType The bounding box hierarchy type to use.
- * @param pic The picture to draw to the renderer.
- * @param numRepeats The number of times to repeat the draw.
- * @param timer The timer used to benchmark the work.
+ * Renders the picture into the renderer. It may or may not use an RTree.
+ * The renderer is chosen upstream. If we want to measure recording, we will
+ * use a RecordPictureRenderer. If we want to measure rendering, we will use a
+ * TiledPictureRenderer.
  */
 static void do_benchmark_work(sk_tools::PictureRenderer* renderer,
-        BBoxType bBoxType,
-        SkPicture* pic,
-        const int numRepeats,
-        BenchTimer* timer) {
-    renderer->setBBoxHierarchyType(bBoxType);
-    renderer->setGridSize(FLAGS_tilesize, FLAGS_tilesize);
+        int benchmarkType, const SkString& path, SkPicture* pic,
+        const int numRepeats, const char *msg, BenchTimer* timer) {
+    SkString msgPrefix;
+
+    switch (benchmarkType){
+        case kNormal_BenchmarkType:
+            msgPrefix.set("Normal");
+            renderer->setBBoxHierarchyType(sk_tools::PictureRenderer::kNone_BBoxHierarchyType);
+            break;
+        case kRTree_BenchmarkType:
+            msgPrefix.set("RTree");
+            renderer->setBBoxHierarchyType(sk_tools::PictureRenderer::kRTree_BBoxHierarchyType);
+            break;
+        default:
+            SkASSERT(0);
+            break;
+    }
+
     renderer->init(pic);
 
-    SkDebugf("%s %d times...\n", renderer->getConfigName().c_str(), numRepeats);
+    /**
+     * If the renderer is not tiled, assume we are measuring recording.
+     */
+    bool isPlayback = (NULL != renderer->getTiledRenderer());
+    // Will be non-null during RTree picture playback. For correctness test.
+    SkBitmap* bitmap = NULL;
+
+    SkDebugf("%s %s %s %d times...\n", msgPrefix.c_str(), msg, path.c_str(), numRepeats);
     for (int i = 0; i < numRepeats; ++i) {
+        // Set up the bitmap.
+        SkBitmap** out = NULL;
+        if (i == 0 && kRTree_BenchmarkType == benchmarkType && isPlayback) {
+            out = &bitmap;
+        }
+
         renderer->setup();
-        // Render once to fill caches
-        renderer->render(NULL, NULL);
+        // Render once to fill caches. Fill bitmap during the first iteration.
+        renderer->render(NULL, out);
         // Render again to measure
         timer->start();
-        renderer->render(NULL);
+        bool result = renderer->render(NULL);
         timer->end();
+
+        // We only care about a false result on playback. RecordPictureRenderer::render will always
+        // return false because we are passing a NULL file name on purpose; which is fine.
+        if (isPlayback && !result) {
+            SkDebugf("Error rendering during playback.\n");
+        }
+    }
+    if (bitmap) {
+        SkAutoTDelete<SkBitmap> bmDeleter(bitmap);
+        if (!compare_picture(path, *bitmap, pic)) {
+            SkDebugf("Error: RTree produced different bitmap\n");
+        }
+    }
+}
+
+/**
+ * Call do_benchmark_work with a tiled renderer using the default tile dimensions.
+ */
+static void benchmark_playback(
+        const BenchmarkControl& benchControl,
+        const SkString& path, SkPicture* pic, BenchTimer* timer) {
+    sk_tools::TiledPictureRenderer renderer;
+
+    SkString message("tiled_playback");
+    message.appendf("_%dx%d", benchControl.fTileSize.fWidth, benchControl.fTileSize.fHeight);
+    do_benchmark_work(&renderer, benchControl.fType,
+            path, pic, kNumPlaybacks, message.c_str(), timer);
+}
+
+/**
+ * Call do_benchmark_work with a RecordPictureRenderer.
+ */
+static void benchmark_recording(
+        const BenchmarkControl& benchControl,
+        const SkString& path, SkPicture* pic, BenchTimer* timer) {
+    sk_tools::RecordPictureRenderer renderer;
+    int numRecordings = 0;
+    switch(benchControl.fType) {
+        case kRTree_BenchmarkType:
+            numRecordings = kNumRTreeRecordings;
+            break;
+        case kNormal_BenchmarkType:
+            numRecordings = kNumNormalRecordings;
+            break;
+    }
+    do_benchmark_work(&renderer, benchControl.fType,
+            path, pic, numRecordings, "recording", timer);
+}
+
+/**
+ * Takes argc,argv along with one of the benchmark functions defined above.
+ * Will loop along all skp files and perform measurments.
+ */
+static void benchmark_loop(
+        int argc,
+        char **argv,
+        const BenchmarkControl& benchControl,
+        SkTArray<Histogram>& histogram) {
+    for (int index = 1; index < argc; ++index) {
+        BenchTimer timer;
+        SkString path(argv[index]);
+        SkAutoTUnref<SkPicture> pic(pic_from_path(path.c_str()));
+        if (NULL == pic) {
+            SkDebugf("Couldn't create picture. Ignoring path: %s\n", path.c_str());
+            continue;
+        }
+        benchControl.fFunction(benchControl, path, pic, &timer);
+        histogram[index - 1].fPath = path;
+        histogram[index - 1].fCpuTime = SkDoubleToScalar(timer.fCpu);
     }
 }
 
 int tool_main(int argc, char** argv);
 int tool_main(int argc, char** argv) {
-    SkCommandLineFlags::Parse(argc, argv);
     SkAutoGraphics ag;
-    bool includeBBoxType[kBBoxTypeCount];
-    for (int bBoxType = 0; bBoxType < kBBoxTypeCount; ++bBoxType) {
-        includeBBoxType[bBoxType] = (FLAGS_bbh.count() == 0) ||
-            FLAGS_bbh.contains(kBBoxHierarchyTypeNames[bBoxType]);
-    }
-    // go through all the pictures
-    SkTArray<Measurement> measurements;
-    for (int index = 0; index < FLAGS_skps.count(); ++index) {
-        const char* path = FLAGS_skps[index];
-        SkPicture* picture = pic_from_path(path);
-        if (NULL == picture) {
-            SkDebugf("Couldn't create picture. Ignoring path: %s\n", path);
-            continue;
-        }
-        SkDebugf("Benchmarking path: %s\n", path);
-        Measurement& measurement = measurements.push_back();
-        measurement.fName = path;
-        for (int bBoxType = 0; bBoxType < kBBoxTypeCount; ++bBoxType) {
-            if (!includeBBoxType[bBoxType]) { continue; }
-            if (FLAGS_playback > 0) {
-                sk_tools::TiledPictureRenderer playbackRenderer;
-                BenchTimer playbackTimer;
-                do_benchmark_work(&playbackRenderer, (BBoxType)bBoxType,
-                                  picture, FLAGS_playback, &playbackTimer);
-                measurement.fPlaybackAverage[bBoxType] = playbackTimer.fCpu;
-            }
-            if (FLAGS_record > 0) {
-                sk_tools::RecordPictureRenderer recordRenderer;
-                BenchTimer recordTimer;
-                do_benchmark_work(&recordRenderer, (BBoxType)bBoxType,
-                                  picture, FLAGS_record, &recordTimer);
-                measurement.fRecordAverage[bBoxType] = recordTimer.fCpu;
-            }
-        }
+    SkString usage;
+    usage.printf("Usage: filename [filename]*\n");
+
+    if (argc < 2) {
+        SkDebugf("%s\n", usage.c_str());
+        return -1;
     }
 
-    Measurement globalMeasurement;
-    for (int bBoxType = 0; bBoxType < kBBoxTypeCount; ++bBoxType) {
-        if (!includeBBoxType[bBoxType]) { continue; }
-        globalMeasurement.fPlaybackAverage[bBoxType] = 0;
-        globalMeasurement.fRecordAverage[bBoxType] = 0;
-        for (int index = 0; index < measurements.count(); ++index) {
-            const Measurement& measurement = measurements[index];
-            globalMeasurement.fPlaybackAverage[bBoxType] +=
-                measurement.fPlaybackAverage[bBoxType];
-            globalMeasurement.fRecordAverage[bBoxType] +=
-                measurement.fRecordAverage[bBoxType];
-        }
-        globalMeasurement.fPlaybackAverage[bBoxType] /= measurements.count();
-        globalMeasurement.fRecordAverage[bBoxType] /= measurements.count();
+    SkTArray<Histogram> histograms[kNumBenchmarks];
+
+    for (size_t i = 0; i < kNumBenchmarks; ++i) {
+        histograms[i].reset(argc - 1);
+        benchmark_loop(argc, argv, BenchmarkControl::Make(i), histograms[i]);
     }
 
     // Output gnuplot readable histogram data..
@@ -144,46 +341,62 @@ int tool_main(int argc, char** argv) {
     SkFILEWStream recordOut(recTitle);
     recordOut.writeText("# ");
     playbackOut.writeText("# ");
-    SkDebugf("---\n");
-    for (int bBoxType = 0; bBoxType < kBBoxTypeCount; ++bBoxType) {
-        if (!includeBBoxType[bBoxType]) { continue; }
+    for (size_t i = 0; i < kNumBenchmarks; ++i) {
         SkString out;
-        out.printf("%s ", kBBoxHierarchyTypeNames[bBoxType]);
-        recordOut.writeText(out.c_str());
-        playbackOut.writeText(out.c_str());
-
-        if (FLAGS_record > 0) {
-            SkDebugf("Average %s recording time: %.3fms\n",
-                kBBoxHierarchyTypeNames[bBoxType],
-                globalMeasurement.fRecordAverage[bBoxType]);
+        out.printf("%s ", BenchmarkControl::GetBenchmarkName(i).c_str());
+        if (BenchmarkControl::GetBenchmarkFunc(i) == &benchmark_recording) {
+            recordOut.writeText(out.c_str());
         }
-        if (FLAGS_playback > 0) {
-            SkDebugf("Average %s playback time: %.3fms\n",
-                kBBoxHierarchyTypeNames[bBoxType],
-                globalMeasurement.fPlaybackAverage[bBoxType]);
+        if (BenchmarkControl::GetBenchmarkFunc(i) == &benchmark_playback) {
+            playbackOut.writeText(out.c_str());
         }
     }
     recordOut.writeText("\n");
     playbackOut.writeText("\n");
     // Write to file, and save recording averages.
-    for (int index = 0; index < measurements.count(); ++index) {
-        const Measurement& measurement = measurements[index];
+    SkScalar avgRecord = SkIntToScalar(0);
+    SkScalar avgPlayback = SkIntToScalar(0);
+    SkScalar avgRecordRTree = SkIntToScalar(0);
+    SkScalar avgPlaybackRTree = SkIntToScalar(0);
+    for (int i = 0; i < argc - 1; ++i) {
         SkString pbLine;
         SkString recLine;
+        // ==== Write record info
+        recLine.printf("%d ", i);
+        SkScalar cpuTime = histograms[BenchmarkControl::kNormalRecord][i].fCpuTime;
+        recLine.appendf("%f ", cpuTime);
+        avgRecord += cpuTime;
+        cpuTime = histograms[BenchmarkControl::kRTreeRecord][i].fCpuTime;
+        recLine.appendf("%f", cpuTime);
+        avgRecordRTree += cpuTime;
+        avgPlaybackRTree += cpuTime;
 
-        pbLine.printf("%d", index);
-        recLine.printf("%d", index);
-        for (int bBoxType = 0; bBoxType < kBBoxTypeCount; ++bBoxType) {
-            if (!includeBBoxType[bBoxType]) { continue; }
-            pbLine.appendf(" %f", measurement.fPlaybackAverage[bBoxType]);
-            recLine.appendf(" %f", measurement.fRecordAverage[bBoxType]);
+        // ==== Write playback info
+        pbLine.printf("%d ", i);
+        pbLine.appendf("%f ", histograms[2][i].fCpuTime);  // Start with normal playback time.
+        avgPlayback += histograms[kNumBbhPlaybackBenchmarks - 1][i].fCpuTime;
+        avgPlaybackRTree += histograms[kNumBbhPlaybackBenchmarks][i].fCpuTime;
+        // Append all playback benchmark times.
+        for (size_t j = kNumBbhPlaybackBenchmarks; j < kNumBenchmarks; ++j) {
+            pbLine.appendf("%f ", histograms[j][i].fCpuTime);
         }
+        pbLine.remove(pbLine.size() - 1, 1);  // Remove trailing space from line.
         pbLine.appendf("\n");
         recLine.appendf("\n");
         playbackOut.writeText(pbLine.c_str());
         recordOut.writeText(recLine.c_str());
     }
+    avgRecord /= argc - 1;
+    avgRecordRTree /= argc - 1;
+    avgPlayback /= argc - 1;
+    avgPlaybackRTree /= argc - 1;
+    SkDebugf("Average base recording time: %.3fms\n", avgRecord);
+    SkDebugf("Average recording time with rtree: %.3fms\n", avgRecordRTree);
+    SkDebugf("Average base playback time: %.3fms\n", avgPlayback);
+    SkDebugf("Average playback time with rtree: %.3fms\n", avgPlaybackRTree);
+
     SkDebugf("\nWrote data to gnuplot-readable files: %s %s\n", pbTitle, recTitle);
+
     return 0;
 }