diff options
Diffstat (limited to 'gm')
8 files changed, 115 insertions, 8 deletions
diff --git a/gm/gm_error.h b/gm/gm_error.h index 16917435d3..e442f5d894 100644 --- a/gm/gm_error.h +++ b/gm/gm_error.h @@ -58,6 +58,22 @@ namespace skiagm { } /** + * Fills in "type" with the ErrorType associated with name "name". + * Returns true if we found one, false if it is an unknown type name. + */ + static bool getErrorTypeByName(const char name[], ErrorType *type) { + for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) { + ErrorType thisType = static_cast<ErrorType>(typeInt); + const char *thisTypeName = getErrorTypeName(thisType); + if (0 == strcmp(thisTypeName, name)) { + *type = thisType; + return true; + } + } + return false; + } + + /** * A combination of 0 or more ErrorTypes. */ class ErrorCombination { @@ -94,6 +110,26 @@ namespace skiagm { } /** + * Returns a string representation of all ErrorTypes in this + * ErrorCombination. + * + * @param separator text with which to separate ErrorType names + */ + SkString asString(const char separator[]) const { + SkString s; + for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) { + ErrorType type = static_cast<ErrorType>(typeInt); + if (this->includes(type)) { + if (!s.isEmpty()) { + s.append(separator); + } + s.append(getErrorTypeName(type)); + } + } + return s; + } + + /** * Returns a new ErrorCombination, which includes the union of all * ErrorTypes in two ErrorCombination objects (this and other). */ diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp index 91aef10188..a91a28fa6c 100644 --- a/gm/gmmain.cpp +++ b/gm/gmmain.cpp @@ -174,13 +174,14 @@ static PipeFlagComboData gPipeWritingFlagCombos[] = { | SkGPipeWriter::kSharedAddressSpace_Flag } }; +const static ErrorCombination kDefaultIgnorableErrorTypes = ErrorCombination() + .plus(kMissingExpectations_ErrorType) + .plus(kIntentionallySkipped_ErrorType); + class GMMain { public: - GMMain() : fUseFileHierarchy(false), fMismatchPath(NULL), fTestsRun(0), - fRenderModesEncountered(1) { - fIgnorableErrorCombination.add(kMissingExpectations_ErrorType); - fIgnorableErrorCombination.add(kIntentionallySkipped_ErrorType); - } + GMMain() : fUseFileHierarchy(false), fIgnorableErrorTypes(kDefaultIgnorableErrorTypes), + fMismatchPath(NULL), fTestsRun(0), fRenderModesEncountered(1) {} SkString make_name(const char shortName[], const char configName[]) { SkString name; @@ -291,7 +292,7 @@ public: int significantErrors = 0; for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) { ErrorType type = static_cast<ErrorType>(typeInt); - if (!fIgnorableErrorCombination.includes(type)) { + if (!fIgnorableErrorTypes.includes(type)) { significantErrors += fFailedTests[type].count(); } } @@ -305,7 +306,7 @@ public: * @param verbose whether to be all verbose about it */ void DisplayResultTypeSummary(ErrorType type, bool verbose) { - bool isIgnorableType = fIgnorableErrorCombination.includes(type); + bool isIgnorableType = fIgnorableErrorTypes.includes(type); SkString line; if (isIgnorableType) { @@ -1105,7 +1106,7 @@ public: // bool fUseFileHierarchy; - ErrorCombination fIgnorableErrorCombination; + ErrorCombination fIgnorableErrorTypes; const char* fMismatchPath; @@ -1209,6 +1210,18 @@ DEFINE_string(gpuCacheSize, "", "<bytes> <count>: Limit the gpu cache to byte si #endif DEFINE_bool(hierarchy, false, "Whether to use multilevel directory structure " "when reading/writing files."); +// TODO(epoger): Maybe should make SkCommandLineFlags handle default string +// values differently, so that the first definition of ignoreErrorTypes worked? +#if 0 +DEFINE_string(ignoreErrorTypes, kDefaultIgnorableErrorTypes.asString(" ").c_str(), + "Space-separated list of ErrorTypes that should be ignored. If any *other* error " + "types are encountered, the tool will exit with a nonzero return value."); +#else +DEFINE_string(ignoreErrorTypes, "", SkString(SkString( + "Space-separated list of ErrorTypes that should be ignored. If any *other* error " + "types are encountered, the tool will exit with a nonzero return value. " + "Defaults to: ") += kDefaultIgnorableErrorTypes.asString(" ")).c_str()); +#endif DEFINE_string(match, "", "Only run tests whose name includes this substring/these substrings " "(more than one can be supplied, separated by spaces)."); DEFINE_string(mismatchPath, "", "Write images for tests that failed due to " @@ -1639,6 +1652,20 @@ int tool_main(int argc, char** argv) { } } + if (FLAGS_ignoreErrorTypes.count() > 0) { + gmmain.fIgnorableErrorTypes = ErrorCombination(); + for (int i = 0; i < FLAGS_ignoreErrorTypes.count(); i++) { + ErrorType type; + const char *name = FLAGS_ignoreErrorTypes[i]; + if (!getErrorTypeByName(name, &type)) { + gm_fprintf(stderr, "cannot find ErrorType with name '%s'\n", name); + return -1; + } else { + gmmain.fIgnorableErrorTypes.add(type); + } + } + } + #if SK_SUPPORT_GPU if (FLAGS_gpuCacheSize.count() > 0) { if (FLAGS_gpuCacheSize.count() != 2) { diff --git a/gm/tests/outputs/ignore-expectations-mismatch/output-expected/command_line b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/command_line new file mode 100644 index 0000000000..f8ffbc8077 --- /dev/null +++ b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/command_line @@ -0,0 +1 @@ +out/Debug/gm --ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/ignore-expectations-mismatch/output-actual/json-summary.txt diff --git a/gm/tests/outputs/ignore-expectations-mismatch/output-expected/json-summary.txt b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/json-summary.txt new file mode 100644 index 0000000000..4cfe15127b --- /dev/null +++ b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/json-summary.txt @@ -0,0 +1,25 @@ +{ + "actual-results" : { + "failed" : { + "565/selftest1" : { + "checksum" : 9512553915271796906 + }, + "8888/selftest1" : { + "checksum" : 14022967492765711532 + } + }, + "failure-ignored" : null, + "no-comparison" : null, + "succeeded" : null + }, + "expected-results" : { + "565/selftest1" : { + "checksums" : [ 11071285354315388429 ], + "ignore-failure" : false + }, + "8888/selftest1" : { + "checksums" : [ 16527650414256125612 ], + "ignore-failure" : false + } + } +} diff --git a/gm/tests/outputs/ignore-expectations-mismatch/output-expected/return_value b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/return_value new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/return_value @@ -0,0 +1 @@ +0 diff --git a/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stderr b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stderr new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stderr diff --git a/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout new file mode 100644 index 0000000000..881c12145a --- /dev/null +++ b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout @@ -0,0 +1,14 @@ +GM: reading expectations from JSON summary file gm/tests/inputs/json/different-pixels.json +GM: drawing... selftest1 [300 200] +GM: Ran 1 GMs +GM: ... over 2 configs ["8888", "565"] +GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"] +GM: ... so there should be a total of 9 tests. +GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 +GM: [ ] 0 NoGpuContext: +GM: [*] 0 IntentionallySkipped: +GM: [*] 0 RenderModeMismatch: +GM: [ ] 2 ExpectationsMismatch: 8888/selftest1 565/selftest1 +GM: [*] 0 MissingExpectations: +GM: [*] 0 WritingReferenceImage: +GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/run.sh b/gm/tests/run.sh index 844294a0a2..0b861b17bc 100755 --- a/gm/tests/run.sh +++ b/gm/tests/run.sh @@ -165,4 +165,7 @@ gm_test "--simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 $ # Confirm that IntentionallySkipped tests are recorded as such. gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUTS/intentionally-skipped-tests" +# Ignore some error types (including ExpectationsMismatch) +gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/ignore-expectations-mismatch" + echo "All tests passed." |