SkQP: Stop requiring users to use posix shell

Also update README.md

Change-Id: I62f5ac38ff4a8c3aa19d895441a76664cb8e8176
Reviewed-on: https://skia-review.googlesource.com/99302
Reviewed-by: Hal Canary <halcanary@google.com>
Commit-Queue: Hal Canary <halcanary@google.com>

8 files changed, 161 insertions, 124 deletions

diff --git a/tools/skqp/README.md b/tools/skqp/README.md
index 7e56411fd4..cf745364db 100644
--- a/tools/skqp/README.md
+++ b/tools/skqp/README.md
@@ -2,80 +2,78 @@
 SkQP
 ====
 
-**Motivation**: Test an Android device's GPU and OpenGLES & Vulkan drivers with
-Skia and Skia's existing unit & rendering tests.
+How to run the SkQP tests
+-------------------------
 
-How To Use SkQP on your Android device:
+1.  Install Chromium's depot\_tools
 
-1.  To build SkQP you need to install the
-    [Android NDK](https://developer.android.com/ndk/).
+        git clone 'https://chromium.googlesource.com/chromium/tools/depot_tools.git'
+        export PATH="${PWD}/depot_tools:${PATH}"
 
-2.  [Checkout depot\_tools and Skia](https://skia.org/user/download),
-    then go to Skia's source directory:
+2.  Install the [Android NDK](https://developer.android.com/ndk/downloads/).
 
-        export PATH="${DEPOT_TOOLS_PATH}:$PATH"
-        cd $SKIA_SOURCE_DIRECTORY
+        cd ~
+        unzip ~/Downloads/android-ndk-*.zip
+        ANDROID_NDK=~/android-ndk-r16b     # Or wherever you installed the Android NDK.
 
-3.  Configure and build Skia for your device's architecture:
+3.  Install the [Android SDK](https://developer.android.com/studio/#command-tools)
 
-        arch='arm64'  # Also valid: 'arm', 'x68', 'x64'
-        android_ndk="${HOME}/android-ndk"  # Or wherever you installed the NDK.
-
-        tools/skqp/generate_gn_args.sh out/${arch}-rel "$android_ndk" $arch
-        tools/git-sync-deps
-        bin/gn gen out/${arch}-rel
-        ninja -C out/${arch}-rel skqp_lib
-
-4.  Download meta.json from [https://goo.gl/jBw3Dd](https://goo.gl/jBw3Dd) .
-    This is the data used to build the validation model.
+        mkdir ~/android-sdk
+        ( cd ~/android-sdk; unzip ~/Downloads/sdk-tools-*.zip )
+        yes | ~/android-sdk/tools/bin/sdkmanager --licenses
+        export ANDROID_HOME=~/android-sdk  # Or wherever you installed the Android SDK.
 
-5.  Generate the validation model data:
+4.  Get the right version of Skia:
 
-        tools/skqp/make_model.sh ~/Downloads/meta.json
+        git clone https://skia.googlesource.com/skia.git
+        cd skia
+        git checkout origin/skqp-dev  # or whatever release tag you need
 
-Run as an executable
---------------------
+5.  Download dependencies, the model, and configure the build.
 
-1.  Build the SkQP program, load files on the device, and run skqp:
+        python tools/skqp/download_model
+        python tools/git-sync-deps
+        python tools/skqp/generate_gn_args out/skqp-arm "$ANDROID_NDK" arm
+        bin/gn gen out/skqp-arm
 
-        ninja -C out/${arch}-rel skqp
-        adb shell "cd /data/local/tmp; rm -rf skqp_assets report"
-        adb push platform_tools/android/apps/skqp/src/main/assets \
-            /data/local/tmp/skqp_assets
-        adb push out/${arch}-rel/skqp /data/local/tmp/
-        adb shell "cd /data/local/tmp; ./skqp skqp_assets report"
+6.  Build, install, and run.
 
-2.  Get the error report if there are errors:
+        platform_tools/android/bin/android_build_app -C out/skqp-arm skqp
+        adb install -r out/skqp-arm/skqp.apk
+        adb logcat -c
+        adb shell am instrument -w org.skia.skqp/android.support.test.runner.AndroidJUnitRunner
 
-        adb pull /data/local/tmp/report /tmp/
-        tools/skqp/sysopen.py /tmp/report/report.html
+7.  Monitor the output with:
 
-Run as an APK
--------------
+        adb logcat org.skia.skqp skia "*:S"
 
-0.  Install the [Android SDK](https://developer.android.com/studio/#command-tools).
+    Note the test's output path on the device.  It will look something like this:
 
-        mkdir ~/android-sdk
-        ( cd ~/android-sdk; unzip ~/Downloads/sdk-tools-*.zip )
-        yes | ~/android-sdk/tools/bin/sdkmanager --licenses
-        export ANDROID_HOME=~/android-sdk
+        01-23 15:22:12.688 27158 27173 I org.skia.skqp:
+        output written to "/storage/emulated/0/Android/data/org.skia.skqp/files/output"
 
-1.  Build the skqp.apk, load it on the device, and run the tests
+8.  Retrieve and view the report with:
 
-        platform_tools/android/bin/android_build_app -C out/${arch}-rel skqp
-        adb install -r out/${arch}-rel/skqp.apk
-        adb logcat -c
-        adb shell am instrument -w \
-            org.skia.skqp/android.support.test.runner.AndroidJUnitRunner
+        OUTPUT_LOCATION="/storage/emulated/0/Android/data/org.skia.skqp/files/output"
+        adb pull $OUTPUT_LOCATION /tmp/
+        tools/skqp/sysopen.py /tmp/output/skqp_report/report.html
 
-2.  Find out where the report went (and look for Skia errors):
+Run as an executable
+--------------------
 
-        adb logcat -d org.skia.skqp skia "*:S"
+1.  Follow steps 1-5 as above.
 
-3.  Retrieve and view the report if there are any errors:
+2.  Build the SkQP program, load files on the device, and run skqp:
 
-        adb pull /storage/emulated/0/Android/data/org.skia.skqp/files/output /tmp/
-        tools/skqp/sysopen.py /tmp/output/skqp_report/report.html
+        ninja -C out/skqp-arm skqp
+        adb shell "cd /data/local/tmp; rm -rf skqp_assets report"
+        adb push platform_tools/android/apps/skqp/src/main/assets \
+            /data/local/tmp/skqp_assets
+        adb push out/skqp-arm/skqp /data/local/tmp/
+        adb shell "cd /data/local/tmp; ./skqp skqp_assets report"
 
+2.  Get and view the error report:
 
+        adb pull /data/local/tmp/report /tmp/
+        tools/skqp/sysopen.py /tmp/report/report.html
 
diff --git a/tools/skqp/download_directory_contents.sh b/tools/skqp/download_directory_contents.sh
deleted file mode 100755
index 1b01a8c0d6..0000000000
--- a/tools/skqp/download_directory_contents.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#! /bin/sh
-
-# Copyright 2018 Google Inc.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-set -e
-
-cd "$(dirname "$0")/../../platform_tools/android/apps/skqp/src/main/assets"
-
-checksum() {
-    [ -f "$1" ] && { md5sum < "$1" | head -c 32; }
-}
-
-download() {
-    if ! [ $1 = "$(checksum "$2")" ]; then
-        mkdir -p "$(dirname "$2")"
-        curl -s -o "$2" "https://storage.googleapis.com/skia-skqp-assets/$1"
-    fi
-}
-
-download $(cat files.checksum) files.txt
-
-COUNT=$(wc -l < files.txt)
-INDEX=1
-SHARD_COUNT=32
-
-cat files.txt | while IFS= read -r LINE; do
-    MD5=$(echo $LINE | awk -F\; '{print $1}')
-    FILENAME=$(echo $LINE | awk -F\; '{print $2}')
-    download $MD5 "$FILENAME" &
-    if [ $(($INDEX % $SHARD_COUNT)) = 0 ]; then
-        wait
-        printf '\r %d / %d ' "$INDEX" "$COUNT"
-    fi
-    INDEX=$(($INDEX + 1))
-done
-printf '\rdone                \n'
-
diff --git a/tools/skqp/download_model b/tools/skqp/download_model
new file mode 100755
index 0000000000..de1aeeddc6
--- /dev/null
+++ b/tools/skqp/download_model
@@ -0,0 +1,68 @@
+#! /usr/bin/env python
+
+# Copyright 2018 Google Inc.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import hashlib
+import os
+import shutil
+import sys
+import threading
+import urllib2
+
+def checksum(path):
+    if not os.path.exists(path):
+        return None
+    m = hashlib.md5()
+    with open(path, 'rb') as f:
+        while True:
+            buf = f.read(4096)
+            if 0 == len(buf):
+                return m.hexdigest()
+            m.update(buf)
+
+def download(md5, path):
+    if not md5 == checksum(path):
+        dirname = os.path.dirname(path)
+        if dirname and not os.path.exists(dirname):
+            try:
+                os.makedirs(dirname)
+            except:
+                # ignore race condition
+                if not os.path.exists(dirname):
+                    raise
+        url = 'https://storage.googleapis.com/skia-skqp-assets/' + md5
+        with open(path, 'wb') as o:
+            shutil.copyfileobj(urllib2.urlopen(url), o)
+
+def main():
+    os.chdir(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
+                         'platform_tools', 'android', 'apps', 'skqp', 'src',
+                         'main', 'assets'))
+    sys.stderr.write('\n    0 / ????   ')
+    with open('files.checksum', 'r') as f:
+        md5 = f.read().strip()
+        assert(len(md5) == 32)
+        download(md5, 'files.txt')
+    with open('files.txt', 'r') as f:
+        records = []
+        for line in f:
+            md5, path = line.strip().split(';', 1)
+            records.append((md5, path))
+    threads = set()
+    sys.stderr.write('\r    0 / %d   ' % len(records))
+    for i, record in enumerate(records):
+        t = threading.Thread(target=download, args=record)
+        t.start()
+        threads.add(t)
+    left = -1
+    while left != 0:
+        count = sum(1 for t in threading.enumerate() if t in threads)
+        if left != count:
+            left = count
+            sys.stderr.write('\r %4d / %d   ' % (len(records) - left, len(records)))
+    sys.stderr.write('\n')
+
+if __name__ == '__main__':
+    main()
diff --git a/tools/skqp/generate_gn_args b/tools/skqp/generate_gn_args
new file mode 100755
index 0000000000..cead4158ad
--- /dev/null
+++ b/tools/skqp/generate_gn_args
@@ -0,0 +1,39 @@
+#! /usr/bin/env python
+
+# Copyright 2018 Google Inc.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+
+fmt = '''ndk = "{ndk}"
+ndk_api = 26
+target_cpu = "{arch}"
+skia_embed_resources = true
+is_debug = false
+skia_enable_pdf = false
+'''
+
+def make_args_gn(out_dir, ndk, arch):
+    if not os.path.exists(out_dir):
+        os.makedirs(out_dir)
+    with open(os.path.join(out_dir, 'args.gn'), 'w') as o:
+        o.write(fmt.format(ndk=os.path.abspath(ndk), arch=arch))
+
+def usage():
+    sys.stderr.write(
+            'Usage:\n' +
+            '  {} TARGET_BUILD_DIR ANDROID_NDK_DIR ARCHITECTURE\n\n'.format(sys.argv[0]) +
+            'ARCHITECTURE should be "arm" "arm64" "x86" or "x64"\n\n')
+    exit(1)
+
+if __name__ == '__main__':
+    if len(sys.argv) != 4:
+        usage()
+    build, android_ndk, arch = sys.argv[1:4]
+    if len(build) == 0 or len(arch) == 0 or not os.path.isdir(android_ndk):
+        usage()
+    make_args_gn(build, android_ndk, arch)
+
+
diff --git a/tools/skqp/generate_gn_args.sh b/tools/skqp/generate_gn_args.sh
deleted file mode 100755
index 8d12fa8513..0000000000
--- a/tools/skqp/generate_gn_args.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#! /bin/sh
-
-# Copyright 2018 Google Inc.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-BUILD="$1"
-ANDROID_NDK="$2"
-ARCH="$3"
-
-if [ $# -ne 3 ] || [ -z "$BUILD" ] || ! [ -d "$ANDROID_NDK" ] || [ -z "$ARCH" ]; then
-    printf 'Usage:\n  %s TARGET_BUILD_DIR ANDROID_NDK_DIR ARCHITECTURE\n\n' "$0" >&2
-    printf 'ARCHITECTURE should be "arm" "arm64" "x86" or "x64"\n\n' >&2
-    exit 1
-fi
-
-ANDROID_NDK="$(cd "$ANDROID_NDK"; pwd)"
-
-mkdir -p "$BUILD"
-
-cat > "$BUILD/args.gn" << EOF
-ndk = "$ANDROID_NDK"
-ndk_api = 26
-target_cpu = "$ARCH"
-skia_embed_resources = true
-is_debug = false
-skia_enable_pdf = false
-EOF
-
-
diff --git a/tools/skqp/gm_knowledge.cpp b/tools/skqp/gm_knowledge.cpp
index 51f0d8f85a..d3de868b24 100644
--- a/tools/skqp/gm_knowledge.cpp
+++ b/tools/skqp/gm_knowledge.cpp
@@ -245,17 +245,18 @@ static constexpr char kDocHead[] =
     "  ac(b, ma(i.src, i));\n"
     "  br(b);\n"
     "  ac(b, ma(t + \"/max.png\", ct(\"max\")));\n"
+    "  ac(b, ct(\" | \"));\n"
     "  ac(b, ma(t + \"/min.png\", ct(\"min\")));\n"
     "  ac(b, ce(\"hr\"));\n"
     "  ac(document.body, b);\n"
     "}\n"
-    "function bar() {\n";
+    "function main() {\n";
 
 static constexpr char kDocTail[] =
     "}\n"
     "</script>\n"
     "</head>\n"
-    "<body onload=\"bar()\">\n"
+    "<body onload=\"main()\">\n"
     "<h1>SkQP Report</h1>\n"
     "<hr>\n"
     "</body>\n"
diff --git a/tools/skqp/make_apk.sh b/tools/skqp/make_apk.sh
index 5c415984cc..6da2063fe3 100755
--- a/tools/skqp/make_apk.sh
+++ b/tools/skqp/make_apk.sh
@@ -33,7 +33,7 @@ cd "$(dirname "$0")/../.."
 
 BUILD=out/skqp-${ARCH}
 
-tools/skqp/generate_gn_args.sh $BUILD "$ANDROID_NDK" $ARCH
+python tools/skqp/generate_gn_args $BUILD "$ANDROID_NDK" $ARCH
 
 GIT_SYNC_DEPS_QUIET=Y tools/git-sync-deps
 
diff --git a/tools/skqp/upload_directory_contents.sh b/tools/skqp/upload_model
index c7f294e69b..c7f294e69b 100755
--- a/tools/skqp/upload_directory_contents.sh
+++ b/tools/skqp/upload_model