diff options
Diffstat (limited to 'tools')
5 files changed, 177 insertions, 8 deletions
diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line index 9530f18c3e..76293588ef 100644 --- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line +++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line @@ -1 +1 @@ -python tools/rebaseline.py --dry-run --json-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --expectations-root tools/tests/rebaseline/input/json1 +python tools/rebaseline.py --expectations-root tools/tests/rebaseline/output/using-json1-expectations/output-actual/gm-expectations --json-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json new file mode 100644 index 0000000000..d7608cd2a0 --- /dev/null +++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json @@ -0,0 +1,52 @@ +{ + "expected-results" : { + "3x3bitmaprect_565.png" : { + "allowed-digests" : null, + "ignore-failure" : false + }, + "3x3bitmaprect_8888.png" : { + "allowed-digests" : null, + "ignore-failure" : false + }, + "aaclip_gpu.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 11899819492385205974 ] + ], + "ignore-failure" : false + }, + "aarectmodes_565.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 14760033689012826769 ] + ], + "ignore-failure" : false + }, + "imageblur_565.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 17796243856503591523 ] + ], + "ignore-failure" : false + }, + "imageblur_8888.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 7426416989687670152 ] + ], + "ignore-failure" : false + }, + "shadertext3_8888.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 10593797161686785561 ] + ], + "ignore-failure" : false + }, + "xfermodeimagefilter_pdf.png" : { + "allowed-digests" : null, + "ignore-failure" : false + }, + "xfermodes_pdf.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 9151974350149210736 ] + ], + "ignore-failure" : false + } + } +} diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-macmini/expected-results.json b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-macmini/expected-results.json new file mode 100644 index 0000000000..79af0cf5e7 --- /dev/null +++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-macmini/expected-results.json @@ -0,0 +1,26 @@ +{ + "expected-results" : { + "aaclip_565.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 12345 ] + ], + "ignore-failure" : false + }, + "aaclip_8888.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 67890 ] + ], + "ignore-failure" : false + }, + "aaclip_gpu.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 11899819492385205974 ] + ], + "ignore-failure" : false + }, + "aaclip_pdf.png" : { + "allowed-digests" : null, + "ignore-failure" : false + } + } +} diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json new file mode 100644 index 0000000000..b314ed68a4 --- /dev/null +++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json @@ -0,0 +1,52 @@ +{ + "expected-results" : { + "3x3bitmaprect_565.png" : { + "allowed-digests" : null, + "ignore-failure" : false + }, + "3x3bitmaprect_8888.png" : { + "allowed-digests" : null, + "ignore-failure" : false + }, + "aaclip_gpu.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 11899819492385205974 ] + ], + "ignore-failure" : false + }, + "aarectmodes_565.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 14760033689012826769 ] + ], + "ignore-failure" : false + }, + "imageblur_565.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 3359963596899141322 ] + ], + "ignore-failure" : false + }, + "imageblur_8888.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 4217923806027861152 ] + ], + "ignore-failure" : false + }, + "shadertext3_8888.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 3713708307125704716 ] + ], + "ignore-failure" : false + }, + "xfermodeimagefilter_pdf.png" : { + "allowed-digests" : null, + "ignore-failure" : false + }, + "xfermodes_pdf.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 9151974350149210736 ] + ], + "ignore-failure" : false + } + } +} diff --git a/tools/tests/run.sh b/tools/tests/run.sh index 3f76de1b58..1a3f40b2a6 100755 --- a/tools/tests/run.sh +++ b/tools/tests/run.sh @@ -39,7 +39,7 @@ function compare_directories { echo "compare_directories requires exactly 2 parameters, got $#" exit 1 fi - diff --exclude=.* $1 $2 + diff --recursive --exclude=.* $1 $2 if [ $? != 0 ]; then echo "failed in: compare_directories $1 $2" exit 1 @@ -129,9 +129,11 @@ function benchgraph_test { compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR } +# Test rebaseline.py's soon-to-disappear image file rebaselining capability. +# # Run rebaseline.py with arguments in $1, recording its dry-run output. # Then compare that dry-run output to the content of $2/output-expected. -function rebaseline_test { +function rebaseline_images_test { if [ $# != 2 ]; then echo "rebaseline_test requires exactly 2 parameters, got $#" exit 1 @@ -150,6 +152,39 @@ function rebaseline_test { compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR } +# Test rebaseline.py's new JSON-format expectations rebaselining capability. +# +# Copy expected-result.json files from $1 into a dir where they can be modified. +# Run rebaseline.py with arguments in $2, recording its output. +# Then compare the output (and modified expected-result.json files) to the +# content of $2/output-expected. +function rebaseline_test { + if [ $# != 3 ]; then + echo "rebaseline_test requires exactly 3 parameters, got $#" + exit 1 + fi + COPY_EXPECTATIONS_FROM_DIR="$1" + ARGS="$2" + ACTUAL_OUTPUT_DIR="$3/output-actual" + EXPECTED_OUTPUT_DIR="$3/output-expected" + + rm -rf $ACTUAL_OUTPUT_DIR + mkdir -p $ACTUAL_OUTPUT_DIR + EXPECTATIONS_TO_MODIFY_DIR="$ACTUAL_OUTPUT_DIR/gm-expectations" + SUBDIRS=$(ls $COPY_EXPECTATIONS_FROM_DIR) + for SUBDIR in $SUBDIRS; do + mkdir -p $EXPECTATIONS_TO_MODIFY_DIR/$SUBDIR + cp $COPY_EXPECTATIONS_FROM_DIR/$SUBDIR/expected-results.json \ + $EXPECTATIONS_TO_MODIFY_DIR/$SUBDIR + done + COMMAND="python tools/rebaseline.py --expectations-root $EXPECTATIONS_TO_MODIFY_DIR $ARGS" + echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line + $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout + echo $? >$ACTUAL_OUTPUT_DIR/return_value + + compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR +} + # Run jsondiff.py with arguments in $1, recording its output. # Then compare that output to the content of $2/output-expected. function jsondiff_test { @@ -230,11 +265,15 @@ fi REBASELINE_INPUT=tools/tests/rebaseline/input REBASELINE_OUTPUT=tools/tests/rebaseline/output -rebaseline_test "--expectations-root fake/expectations/path --json-base-url file:$REBASELINE_INPUT/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/subset" -rebaseline_test "--json-base-url file:nonexistent-path --tests test1 test2" "$REBASELINE_OUTPUT/all" -rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1" -rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new" "$REBASELINE_OUTPUT/using-json1-add-new" -rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --expectations-root $REBASELINE_INPUT/json1" "$REBASELINE_OUTPUT/using-json1-expectations" + +# These test the old image-file expectations. +rebaseline_images_test "--expectations-root fake/expectations/path --json-base-url file:$REBASELINE_INPUT/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/subset" +rebaseline_images_test "--json-base-url file:nonexistent-path --tests test1 test2" "$REBASELINE_OUTPUT/all" +rebaseline_images_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1" +rebaseline_images_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new" "$REBASELINE_OUTPUT/using-json1-add-new" + +# These test the new JSON-format expectations. +rebaseline_test "$REBASELINE_INPUT/json1" "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1-expectations" # # Test jsondiff.py ... |