diff options
| author |  Hal Canary <halcanary@google.com> | 2018-01-23 14:44:27 -0500 |
|---|---|---|
| committer |  Skia Commit-Bot <skia-commit-bot@chromium.org> | 2018-01-23 19:46:39 +0000 |
| commit | 8d4a9f01f672cf78246e5ea4fcb5f04bffce84a2 (patch) | |
| tree | c3aca2b014af173bb5cb64fe9f80bd1b7a1eddd5 /tools/skqp | |
| parent | 16b56f5dcf65b384df4bb191862560d5d6e8e514 (diff) | |
SkQP: upload_directory_contents.sh is sharded
NOTRY=true
Change-Id: I1976778cd2c2bdc82f551285b3d36212034f8f7a
Reviewed-on: https://skia-review.googlesource.com/98643
Reviewed-by: Hal Canary <halcanary@google.com>
Commit-Queue: Hal Canary <halcanary@google.com>
Diffstat (limited to 'tools/skqp')
| -rwxr-xr-x | tools/skqp/upload_directory_contents.sh | 12 |
1 files changed, 9 insertions, 3 deletions
diff --git a/tools/skqp/upload_directory_contents.sh b/tools/skqp/upload_directory_contents.sh index b4bff31dc5..b3dc4456aa 100755 --- a/tools/skqp/upload_directory_contents.sh +++ b/tools/skqp/upload_directory_contents.sh @@ -13,7 +13,7 @@ upload() { MD5=$(md5sum < "$1" | head -c 32) if ! grep -q "$MD5" "$EXTANT"; then URL="gs://skia-skqp-assets/$MD5" - gsutil cp "$1" "$URL" > /dev/null 2>&1 + gsutil cp "$1" "$URL" > /dev/null 2>&1 & fi echo $MD5 } @@ -28,9 +28,10 @@ FILES="$(mktemp "${TMPDIR:-/tmp}/files.XXXXXXXXXX")" COUNT=$(find * -type f | wc -l) INDEX=1 +SHARD_COUNT=32 + find * -type f | sort | while IFS= read -r FILENAME; do printf '\r %d / %d ' "$INDEX" "$COUNT" - INDEX=$(( $INDEX + 1)) if ! [ -f "$FILENAME" ]; then echo error [${FILENAME}] >&2; exit 1; @@ -38,8 +39,13 @@ find * -type f | sort | while IFS= read -r FILENAME; do case "$FILENAME" in *\;*) echo bad filename: $FILENAME >&2; exit 1;; esac MD5=$(upload "$FILENAME") printf '%s;%s\n' "$MD5" "$FILENAME" >> "$FILES" -done + if [ $(($INDEX % $SHARD_COUNT)) = 0 ]; then + wait + fi + INDEX=$(( $INDEX + 1)) +done +printf '\rdone \n' upload "$FILES" > files.checksum rm "$FILES" "$EXTANT" |