diff options
15 files changed, 280 insertions, 21 deletions
diff --git a/gm/gm_expectations.h b/gm/gm_expectations.h index 07f0e60dbd..acff9d478b 100644 --- a/gm/gm_expectations.h +++ b/gm/gm_expectations.h @@ -10,9 +10,11 @@ #include "gm.h" #include "SkBitmap.h" #include "SkBitmapChecksummer.h" +#include "SkData.h" #include "SkImageDecoder.h" #include "SkOSFile.h" #include "SkRefCnt.h" +#include "SkStream.h" #include "SkTArray.h" #ifdef SK_BUILD_FOR_WIN @@ -21,11 +23,25 @@ #pragma warning(push) #pragma warning(disable : 4530) #endif +#include "json/reader.h" #include "json/value.h" #ifdef SK_BUILD_FOR_WIN #pragma warning(pop) #endif +#define DEBUGFAIL_SEE_STDERR SkDEBUGFAIL("see stderr for message") + +const static char kJsonKey_ActualResults[] = "actual-results"; +const static char kJsonKey_ActualResults_Failed[] = "failed"; +const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored"; +const static char kJsonKey_ActualResults_NoComparison[] = "no-comparison"; +const static char kJsonKey_ActualResults_Succeeded[] = "succeeded"; +const static char kJsonKey_ActualResults_AnyStatus_Checksum[] = "checksum"; + +const static char kJsonKey_ExpectedResults[] = "expected-results"; +const static char kJsonKey_ExpectedResults_Checksums[] = "checksums"; +const static char kJsonKey_ExpectedResults_IgnoreFailure[] = "ignore-failure"; + namespace skiagm { // The actual type we use to represent a checksum is hidden in here. @@ -33,6 +49,9 @@ namespace skiagm { static inline Json::Value asJsonValue(Checksum checksum) { return checksum; } + static inline Checksum asChecksum(Json::Value jsonValue) { + return jsonValue.asUInt64(); + } static SkString make_filename(const char path[], const char renderModeDescriptor[], @@ -54,27 +73,69 @@ namespace skiagm { public: /** * No expectations at all. - * - * We set ignoreFailure to false by default, but it doesn't really - * matter... the result will always be "no-comparison" anyway. */ - Expectations(bool ignoreFailure=false) { + Expectations(bool ignoreFailure=kDefaultIgnoreFailure) { fIgnoreFailure = ignoreFailure; } /** * Expect exactly one image (appropriate for the case when we * are comparing against a single PNG file). - * - * By default, DO NOT ignore failures. */ - Expectations(const SkBitmap& bitmap, bool ignoreFailure=false) { + Expectations(const SkBitmap& bitmap, bool ignoreFailure=kDefaultIgnoreFailure) { fBitmap = bitmap; fIgnoreFailure = ignoreFailure; fAllowedChecksums.push_back() = SkBitmapChecksummer::Compute64(bitmap); } /** + * Create Expectations from a JSON element as found within the + * kJsonKey_ExpectedResults section. + * + * It's fine if the jsonElement is null or empty; in that case, we just + * don't have any expectations. + */ + Expectations(Json::Value jsonElement) { + if (jsonElement.empty()) { + fIgnoreFailure = kDefaultIgnoreFailure; + } else { + Json::Value ignoreFailure = jsonElement[kJsonKey_ExpectedResults_IgnoreFailure]; + if (ignoreFailure.isNull()) { + fIgnoreFailure = kDefaultIgnoreFailure; + } else if (!ignoreFailure.isBool()) { + fprintf(stderr, "found non-boolean json value for key '%s' in element '%s'\n", + kJsonKey_ExpectedResults_IgnoreFailure, + jsonElement.toStyledString().c_str()); + DEBUGFAIL_SEE_STDERR; + fIgnoreFailure = kDefaultIgnoreFailure; + } else { + fIgnoreFailure = ignoreFailure.asBool(); + } + + Json::Value allowedChecksums = jsonElement[kJsonKey_ExpectedResults_Checksums]; + if (allowedChecksums.isNull()) { + // ok, we'll just assume there aren't any expected checksums to compare against + } else if (!allowedChecksums.isArray()) { + fprintf(stderr, "found non-array json value for key '%s' in element '%s'\n", + kJsonKey_ExpectedResults_Checksums, + jsonElement.toStyledString().c_str()); + DEBUGFAIL_SEE_STDERR; + } else { + for (Json::ArrayIndex i=0; i<allowedChecksums.size(); i++) { + Json::Value checksumElement = allowedChecksums[i]; + if (!checksumElement.isIntegral()) { + fprintf(stderr, "found non-integer checksum in json element '%s'\n", + jsonElement.toStyledString().c_str()); + DEBUGFAIL_SEE_STDERR; + } else { + fAllowedChecksums.push_back() = asChecksum(checksumElement); + } + } + } + } + } + + /** * Returns true iff we want to ignore failed expectations. */ bool ignoreFailure() const { return this->fIgnoreFailure; } @@ -126,6 +187,8 @@ namespace skiagm { } private: + const static bool kDefaultIgnoreFailure = false; + SkTArray<Checksum> fAllowedChecksums; bool fIgnoreFailure; SkBitmap fBitmap; @@ -181,5 +244,119 @@ namespace skiagm { const bool fNotifyOfMissingFiles; }; + /** + * Return Expectations based on JSON summary file. + */ + class JsonExpectationsSource : public ExpectationsSource { + public: + /** + * Create an ExpectationsSource that will return Expectations based on + * a JSON file. + * + * jsonPath: path to JSON file to read + */ + JsonExpectationsSource(const char *jsonPath) { + parse(jsonPath, &fJsonRoot); + fJsonExpectedResults = fJsonRoot[kJsonKey_ExpectedResults]; + } + + Expectations get(const char *testName) SK_OVERRIDE { + return Expectations(fJsonExpectedResults[testName]); + } + + private: + + /** + * Read as many bytes as possible (up to maxBytes) from the stream into + * an SkData object. + * + * If the returned SkData contains fewer than maxBytes, then EOF has been + * reached and no more data would be available from subsequent calls. + * (If EOF has already been reached, then this call will return an empty + * SkData object immediately.) + * + * If there are fewer than maxBytes bytes available to read from the + * stream, but the stream has not been closed yet, this call will block + * until there are enough bytes to read or the stream has been closed. + * + * It is up to the caller to call unref() on the returned SkData object + * once the data is no longer needed, so that the underlying buffer will + * be freed. For example: + * + * { + * size_t maxBytes = 256; + * SkAutoDataUnref dataRef(readIntoSkData(stream, maxBytes)); + * if (NULL != dataRef.get()) { + * size_t bytesActuallyRead = dataRef.get()->size(); + * // use the data... + * } + * } + * // underlying buffer has been freed, thanks to auto unref + * + */ + // TODO(epoger): Move this, into SkStream.[cpp|h] as attempted in + // https://codereview.appspot.com/7300071 ? + // And maybe readFileIntoSkData() also? + static SkData* readIntoSkData(SkStream &stream, size_t maxBytes) { + if (0 == maxBytes) { + return SkData::NewEmpty(); + } + char* bufStart = reinterpret_cast<char *>(sk_malloc_throw(maxBytes)); + char* bufPtr = bufStart; + size_t bytesRemaining = maxBytes; + while (bytesRemaining > 0) { + size_t bytesReadThisTime = stream.read(bufPtr, bytesRemaining); + if (0 == bytesReadThisTime) { + break; + } + bytesRemaining -= bytesReadThisTime; + bufPtr += bytesReadThisTime; + } + return SkData::NewFromMalloc(bufStart, maxBytes - bytesRemaining); + } + + /** + * Wrapper around readIntoSkData for files: reads the entire file into + * an SkData object. + */ + static SkData* readFileIntoSkData(SkFILEStream &stream) { + return readIntoSkData(stream, stream.getLength()); + } + + /** + * Read the file contents from jsonPath and parse them into jsonRoot. + * + * Returns true if successful. + */ + static bool parse(const char *jsonPath, Json::Value *jsonRoot) { + SkFILEStream inFile(jsonPath); + if (!inFile.isValid()) { + fprintf(stderr, "unable to read JSON file %s\n", jsonPath); + DEBUGFAIL_SEE_STDERR; + return false; + } + + SkAutoDataUnref dataRef(readFileIntoSkData(inFile)); + if (NULL == dataRef.get()) { + fprintf(stderr, "error reading JSON file %s\n", jsonPath); + DEBUGFAIL_SEE_STDERR; + return false; + } + + const char *bytes = reinterpret_cast<const char *>(dataRef.get()->data()); + size_t size = dataRef.get()->size(); + Json::Reader reader; + if (!reader.parse(bytes, bytes+size, *jsonRoot)) { + fprintf(stderr, "error parsing JSON file %s\n", jsonPath); + DEBUGFAIL_SEE_STDERR; + return false; + } + return true; + } + + Json::Value fJsonRoot; + Json::Value fJsonExpectedResults; + }; + } #endif diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp index da7b79e70b..a3e28338a6 100644 --- a/gm/gmmain.cpp +++ b/gm/gmmain.cpp @@ -94,17 +94,6 @@ const static ErrorBitfield ERROR_IMAGE_MISMATCH = 0x02; const static ErrorBitfield ERROR_READING_REFERENCE_IMAGE = 0x08; const static ErrorBitfield ERROR_WRITING_REFERENCE_IMAGE = 0x10; -const static char kJsonKey_ActualResults[] = "actual-results"; -const static char kJsonKey_ActualResults_Failed[] = "failed"; -const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored"; -const static char kJsonKey_ActualResults_NoComparison[] = "no-comparison"; -const static char kJsonKey_ActualResults_Succeeded[] = "succeeded"; -const static char kJsonKey_ActualResults_AnyStatus_Checksum[] = "checksum"; - -const static char kJsonKey_ExpectedResults[] = "expected-results"; -const static char kJsonKey_ExpectedResults_Checksums[] = "checksums"; -const static char kJsonKey_ExpectedResults_IgnoreFailure[] = "ignore-failure"; - using namespace skiagm; struct FailRec { @@ -1392,10 +1381,10 @@ int tool_main(int argc, char** argv) { IndividualImageExpectationsSource, (readPath, notifyMissingReadReference))); } else { - fprintf(stderr, "reading expectations from JSON summary file %s ", + fprintf(stderr, "reading expectations from JSON summary file %s\n", readPath); - fprintf(stderr, "BUT WE DON'T KNOW HOW TO DO THIS YET!\n"); - return -1; + gmmain.fExpectationsSource.reset(SkNEW_ARGS( + JsonExpectationsSource, (readPath))); } } if (writePath) { diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line new file mode 100644 index 0000000000..1226cf28fc --- /dev/null +++ b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line @@ -0,0 +1 @@ +out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummary gm/tests/outputs/compared-against-different-pixels-json/output-actual/json-summary.txt diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/json-summary.txt new file mode 100644 index 0000000000..37709e1ffc --- /dev/null +++ b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/json-summary.txt @@ -0,0 +1,25 @@ +{ + "actual-results" : { + "failed" : { + "565/dashing2" : { + "checksum" : FAKE + }, + "8888/dashing2" : { + "checksum" : FAKE + } + }, + "failure-ignored" : null, + "no-comparison" : null, + "succeeded" : null + }, + "expected-results" : { + "565/dashing2" : { + "checksums" : [ FAKE ], + "ignore-failure" : false + }, + "8888/dashing2" : { + "checksums" : [ FAKE ], + "ignore-failure" : false + } + } +} diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/return_value b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/return_value new file mode 100644 index 0000000000..ace9d03621 --- /dev/null +++ b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/return_value @@ -0,0 +1 @@ +255 diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout new file mode 100644 index 0000000000..9194b40bf5 --- /dev/null +++ b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout @@ -0,0 +1,3 @@ +reading expectations from JSON summary file gm/tests/inputs/json/different-pixels.json +drawing... dashing2 [640 480] +Ran 1 tests: 0 passed, 1 failed, 0 missing reference images diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line new file mode 100644 index 0000000000..e8fb0b97e6 --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line @@ -0,0 +1 @@ +out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/json/identical-bytes.json --writeJsonSummary gm/tests/outputs/compared-against-identical-bytes-json/output-actual/json-summary.txt diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/json-summary.txt new file mode 100644 index 0000000000..bed3b2d3b4 --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/json-summary.txt @@ -0,0 +1,25 @@ +{ + "actual-results" : { + "failed" : null, + "failure-ignored" : null, + "no-comparison" : null, + "succeeded" : { + "565/dashing2" : { + "checksum" : FAKE + }, + "8888/dashing2" : { + "checksum" : FAKE + } + } + }, + "expected-results" : { + "565/dashing2" : { + "checksums" : [ FAKE ], + "ignore-failure" : false + }, + "8888/dashing2" : { + "checksums" : [ FAKE ], + "ignore-failure" : false + } + } +} diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/return_value b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/return_value new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/return_value @@ -0,0 +1 @@ +0 diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout new file mode 100644 index 0000000000..afa854c231 --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout @@ -0,0 +1,3 @@ +reading expectations from JSON summary file gm/tests/inputs/json/identical-bytes.json +drawing... dashing2 [640 480] +Ran 1 tests: 1 passed, 0 failed, 0 missing reference images diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line new file mode 100644 index 0000000000..a596064846 --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line @@ -0,0 +1 @@ +out/Debug/gm --hierarchy --match dashing2 --config 8888 --config 565 -r gm/tests/inputs/json/identical-pixels.json --writeJsonSummary gm/tests/outputs/compared-against-identical-pixels-json/output-actual/json-summary.txt diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/json-summary.txt new file mode 100644 index 0000000000..bed3b2d3b4 --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/json-summary.txt @@ -0,0 +1,25 @@ +{ + "actual-results" : { + "failed" : null, + "failure-ignored" : null, + "no-comparison" : null, + "succeeded" : { + "565/dashing2" : { + "checksum" : FAKE + }, + "8888/dashing2" : { + "checksum" : FAKE + } + } + }, + "expected-results" : { + "565/dashing2" : { + "checksums" : [ FAKE ], + "ignore-failure" : false + }, + "8888/dashing2" : { + "checksums" : [ FAKE ], + "ignore-failure" : false + } + } +} diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/return_value b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/return_value new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/return_value @@ -0,0 +1 @@ +0 diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout new file mode 100644 index 0000000000..3b2671aad5 --- /dev/null +++ b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout @@ -0,0 +1,3 @@ +reading expectations from JSON summary file gm/tests/inputs/json/identical-pixels.json +drawing... dashing2 [640 480] +Ran 1 tests: 1 passed, 0 failed, 0 missing reference images diff --git a/gm/tests/run.sh b/gm/tests/run.sh index e1b6ce7707..a90db38b2b 100755 --- a/gm/tests/run.sh +++ b/gm/tests/run.sh @@ -145,12 +145,15 @@ create_inputs_dir $GM_INPUTS # Compare generated image against an input image file with identical bytes. gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes-images" +gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json" # Compare generated image against an input image file with identical pixels but different PNG encoding. gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images" +gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json" # Compare generated image against an input image file with different pixels. gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images" +gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json" # Compare generated image against an empty "expected image" dir. gm_test "--hierarchy --match dashing2 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir" |