diff options
15 files changed, 168 insertions, 11 deletions
diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp index b2cc8d8fb4..2f18aaacf7 100644 --- a/gm/gmmain.cpp +++ b/gm/gmmain.cpp @@ -15,6 +15,7 @@ #include "gm.h" #include "system_preferences.h" +#include "SkBitmapChecksummer.h" #include "SkColorPriv.h" #include "SkData.h" #include "SkDeferredCanvas.h" @@ -32,6 +33,8 @@ #include "SkTileGridPicture.h" #include "SamplePipeControllers.h" +#include "json/value.h" + #if SK_SUPPORT_GPU #include "GrContextFactory.h" #include "GrRenderTarget.h" @@ -74,6 +77,16 @@ const static ErrorBitfield ERROR_DIMENSION_MISMATCH = 0x04; const static ErrorBitfield ERROR_READING_REFERENCE_IMAGE = 0x08; const static ErrorBitfield ERROR_WRITING_REFERENCE_IMAGE = 0x10; +const static char kJsonKey_ActualResults[] = "actual-results"; +const static char kJsonKey_ActualResults_Failed[] = "failed"; +const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored"; +const static char kJsonKey_ActualResults_Succeeded[] = "succeeded"; +const static char kJsonKey_ActualResults_AnyStatus_Checksum[] = "checksum"; + +const static char kJsonKey_ExpectedResults[] = "expected-results"; +const static char kJsonKey_ExpectedResults_Checksums[] = "checksums"; +const static char kJsonKey_ExpectedResults_IgnoreFailure[] = "ignore-failure"; + using namespace skiagm; /* @@ -570,24 +583,57 @@ public: ErrorBitfield compare_to_reference_image_on_disk( const char readPath [], const SkString& name, SkBitmap &bitmap, const char diffPath [], const char renderModeDescriptor []) { + ErrorBitfield retval; SkString path = make_filename(readPath, "", name, "png"); SkBitmap referenceBitmap; - if (SkImageDecoder::DecodeFile(path.c_str(), &referenceBitmap, + Json::Value expectedChecksumsArray; + + bool decodedReferenceBitmap = + SkImageDecoder::DecodeFile(path.c_str(), &referenceBitmap, SkBitmap::kARGB_8888_Config, SkImageDecoder::kDecodePixels_Mode, - NULL)) { - return compare_to_reference_image_in_memory(name, bitmap, - referenceBitmap, - diffPath, - renderModeDescriptor); + NULL); + if (decodedReferenceBitmap) { + expectedChecksumsArray.append(Json::UInt64( + SkBitmapChecksummer::Compute64(referenceBitmap))); + retval = compare_to_reference_image_in_memory(name, bitmap, + referenceBitmap, + diffPath, + renderModeDescriptor); } else { if (fNotifyMissingReadReference) { fprintf(stderr, "FAILED to read %s\n", path.c_str()); } RecordError(ERROR_READING_REFERENCE_IMAGE, name, renderModeDescriptor); - return ERROR_READING_REFERENCE_IMAGE; + retval = ERROR_READING_REFERENCE_IMAGE; + } + + // Add this result to the appropriate JSON collection of actual results, + // depending on status. + Json::Value actualResults; + actualResults[kJsonKey_ActualResults_AnyStatus_Checksum] = Json::UInt64( + SkBitmapChecksummer::Compute64(bitmap)); + if (decodedReferenceBitmap) { + if (ERROR_NONE == retval) { + fJsonActualResults_Succeeded[name.c_str()] = actualResults; + } else { + fJsonActualResults_Failed[name.c_str()] = actualResults; + } + } else { + fJsonActualResults_FailureIgnored[name.c_str()] = actualResults; } + + // Add this test to the JSON collection of expected results. + // For now, we assume that this collection starts out empty and we + // just fill it in as we go; once gm accepts a JSON file as input, + // we'll have to change that. + Json::Value expectedResults; + expectedResults[kJsonKey_ExpectedResults_Checksums] = expectedChecksumsArray; + expectedResults[kJsonKey_ExpectedResults_IgnoreFailure] = !decodedReferenceBitmap; + fJsonExpectedResults[name.c_str()] = expectedResults; + + return retval; } // NOTE: As far as I can tell, this function is NEVER called with a @@ -791,6 +837,11 @@ public: // information about all failed tests we have encountered so far SkTArray<FailRec> fFailedTests; + Json::Value fJsonExpectedResults; + Json::Value fJsonActualResults_Failed; + Json::Value fJsonActualResults_FailureIgnored; + Json::Value fJsonActualResults_Succeeded; + }; // end of GMMain class definition #if SK_SUPPORT_GPU @@ -874,6 +925,7 @@ static void usage(const char * argv0) { " [--notexturecache]: disable the gpu texture cache\n" " [--tiledPipe]: Exercise tiled SkGPipe replay\n" " [--tileGrid]: use a tileGrid structure for SkPicture testing\n" +" [--writeJsonSummary <path>]: write a JSON-formatted result summary to this file\n" " [--writePath|-w <path>]: write rendered images into this directory\n" " [--writePicturePath|-wp <path>]: write .skp files into this directory\n" ); @@ -957,6 +1009,7 @@ int tool_main(int argc, char** argv) { setSystemPreferences(); GMMain gmmain; + const char* writeJsonSummaryPath = NULL;// if non-null, where we write the JSON summary const char* writePath = NULL; // if non-null, where we write the originals const char* writePicturePath = NULL; // if non-null, where we write serialized pictures const char* readPath = NULL; // if non-null, were we read from to compare @@ -1080,6 +1133,11 @@ int tool_main(int argc, char** argv) { if (argv < stop && **argv) { writePath = *argv; } + } else if (0 == strcmp(*argv, "--writeJsonSummary")) { + argv++; + if (argv < stop && **argv) { + writeJsonSummaryPath = *argv; + } } else if ((0 == strcmp(*argv, "--writePicturePath")) || (0 == strcmp(*argv, "-wp"))) { argv++; @@ -1351,6 +1409,22 @@ int tool_main(int argc, char** argv) { testsRun, testsPassed, testsFailed, testsMissingReferenceImages); gmmain.ListErrors(); + if (NULL != writeJsonSummaryPath) { + Json::Value actualResults; + actualResults[kJsonKey_ActualResults_Failed] = + gmmain.fJsonActualResults_Failed; + actualResults[kJsonKey_ActualResults_FailureIgnored] = + gmmain.fJsonActualResults_FailureIgnored; + actualResults[kJsonKey_ActualResults_Succeeded] = + gmmain.fJsonActualResults_Succeeded; + Json::Value root; + root[kJsonKey_ActualResults] = actualResults; + root[kJsonKey_ExpectedResults] = gmmain.fJsonExpectedResults; + std::string jsonStdString = root.toStyledString(); + SkFILEWStream stream(writeJsonSummaryPath); + stream.write(jsonStdString.c_str(), jsonStdString.length()); + } + #if SK_SUPPORT_GPU #if GR_CACHE_STATS diff --git a/gm/tests/inputs/empty-dir/README b/gm/tests/inputs/empty-dir/README new file mode 100644 index 0000000000..4d39134f2b --- /dev/null +++ b/gm/tests/inputs/empty-dir/README @@ -0,0 +1 @@ +This directory intentionally left empty. Except for this file. diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line index ef8c5afda4..4253e730ef 100644 --- a/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line +++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line @@ -1 +1 @@ -out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/different-pixels -w gm/tests/outputs/compared-against-different-pixels/output-actual/images +out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/different-pixels --writeJsonSummary gm/tests/outputs/compared-against-different-pixels/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-different-pixels/output-actual/images diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt new file mode 100644 index 0000000000..179c60c162 --- /dev/null +++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt @@ -0,0 +1,17 @@ +{ + "actual-results" : { + "failed" : { + "8888/dashing2" : { + "checksum" : 2675870163990933333 + } + }, + "failure-ignored" : null, + "succeeded" : null + }, + "expected-results" : { + "8888/dashing2" : { + "checksums" : [ 15161495552186645995 ], + "ignore-failure" : false + } + } +} diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line new file mode 100644 index 0000000000..0ee96beddd --- /dev/null +++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line @@ -0,0 +1 @@ +out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/empty-dir --writeJsonSummary gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-empty-dir/output-actual/images diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/images/8888/dashing2.png b/gm/tests/outputs/compared-against-empty-dir/output-expected/images/8888/dashing2.png Binary files differnew file mode 100644 index 0000000000..465c0199df --- /dev/null +++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/images/8888/dashing2.png diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt new file mode 100644 index 0000000000..935ba5037d --- /dev/null +++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt @@ -0,0 +1,17 @@ +{ + "actual-results" : { + "failed" : null, + "failure-ignored" : { + "8888/dashing2" : { + "checksum" : 2675870163990933333 + } + }, + "succeeded" : null + }, + "expected-results" : { + "8888/dashing2" : { + "checksums" : null, + "ignore-failure" : true + } + } +} diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/return_value b/gm/tests/outputs/compared-against-empty-dir/output-expected/return_value new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/return_value @@ -0,0 +1 @@ +0 diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout new file mode 100644 index 0000000000..436554d29e --- /dev/null +++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout @@ -0,0 +1,5 @@ +reading from gm/tests/inputs/empty-dir +writing to gm/tests/outputs/compared-against-empty-dir/output-actual/images +drawing... dashing2 [640 480] +FAILED to read gm/tests/inputs/empty-dir/8888/dashing2.png +Ran 1 tests: 0 passed, 0 failed, 1 missing reference images diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line index 6e89999781..556498ab12 100644 --- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line +++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line @@ -1 +1 @@ -out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-bytes -w gm/tests/outputs/compared-against-identical-bytes/output-actual/images +out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-bytes --writeJsonSummary gm/tests/outputs/compared-against-identical-bytes/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-identical-bytes/output-actual/images diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt new file mode 100644 index 0000000000..cbb28e1172 --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt @@ -0,0 +1,17 @@ +{ + "actual-results" : { + "failed" : null, + "failure-ignored" : null, + "succeeded" : { + "8888/dashing2" : { + "checksum" : 2675870163990933333 + } + } + }, + "expected-results" : { + "8888/dashing2" : { + "checksums" : [ 2675870163990933333 ], + "ignore-failure" : false + } + } +} diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line index 285cef2125..ecdef271da 100644 --- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line +++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line @@ -1 +1 @@ -out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-pixels -w gm/tests/outputs/compared-against-identical-pixels/output-actual/images +out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-pixels --writeJsonSummary gm/tests/outputs/compared-against-identical-pixels/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-identical-pixels/output-actual/images diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt new file mode 100644 index 0000000000..cbb28e1172 --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt @@ -0,0 +1,17 @@ +{ + "actual-results" : { + "failed" : null, + "failure-ignored" : null, + "succeeded" : { + "8888/dashing2" : { + "checksum" : 2675870163990933333 + } + } + }, + "expected-results" : { + "8888/dashing2" : { + "checksums" : [ 2675870163990933333 ], + "ignore-failure" : false + } + } +} diff --git a/gm/tests/run.sh b/gm/tests/run.sh index b97b28904b..2bb9441d46 100755 --- a/gm/tests/run.sh +++ b/gm/tests/run.sh @@ -38,6 +38,7 @@ function compare_directories { # - with the arguments in $1 # - writing resulting images into $2/output-actual/images # - writing stdout into $2/output-actual/stdout +# - writing json summary into $2/output-actual/json-summary.txt # - writing return value into $2/output-actual/return_value # Then compare all of those against $2/output-expected . function gm_test { @@ -51,7 +52,7 @@ function gm_test { rm -rf $ACTUAL_OUTPUT_DIR mkdir -p $ACTUAL_OUTPUT_DIR - COMMAND="$GM_BINARY $GM_ARGS -w $ACTUAL_OUTPUT_DIR/images" + COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummary $ACTUAL_OUTPUT_DIR/json-summary.txt -w $ACTUAL_OUTPUT_DIR/images" echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout echo $? >$ACTUAL_OUTPUT_DIR/return_value @@ -72,4 +73,7 @@ gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/identical-pixe # Compare generated image against an input image file with different pixels. gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels" +# Compare generated image against an empty "expected image" dir. +gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir" + echo "All tests passed." diff --git a/gyp/gm.gyp b/gyp/gm.gyp index 6f0fc3d377..2dd9aa3c8c 100644 --- a/gyp/gm.gyp +++ b/gyp/gm.gyp @@ -10,6 +10,7 @@ 'include_dirs' : [ '../src/core', '../src/pipe/utils/', + '../src/utils/', ], 'includes': [ 'gmslides.gypi', @@ -26,6 +27,8 @@ 'effects.gyp:effects', 'images.gyp:images', 'pdf.gyp:pdf', + 'utils.gyp:utils', + '../third_party/externals/jsoncpp/jsoncpp.gyp:jsoncpp', ], 'conditions': [ ['skia_os == "mac"', { |
