From 122b38560f2f2a876e009485efba318961491699 Mon Sep 17 00:00:00 2001 From: Ben Wagner Date: Fri, 20 Apr 2018 13:43:23 -0400 Subject: Add SwiftShader Test jobs. Bug: skia:7671 Change-Id: I29134929071ba6c59de948e5f579d33e973f9baa Reviewed-on: https://skia-review.googlesource.com/122541 Auto-Submit: Ben Wagner Commit-Queue: Ben Wagner Reviewed-by: Kevin Lubick --- infra/bots/jobs.json | 2 + ...-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader.json | 319 ++++++++++++++++ infra/bots/recipe_modules/flavor/examples/full.py | 1 + infra/bots/recipe_modules/flavor/gn_flavor.py | 3 + ...CE-CPU-AVX2-x86_64-Release-All-SwiftShader.json | 287 ++++++++++++++ infra/bots/recipes/test.py | 7 +- infra/bots/tasks.json | 414 +++++++++++++++++++++ 7 files changed, 1032 insertions(+), 1 deletion(-) create mode 100644 infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader.json create mode 100644 infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader.json (limited to 'infra') diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json index 366e033f20..209baf7982 100644 --- a/infra/bots/jobs.json +++ b/infra/bots/jobs.json @@ -399,6 +399,7 @@ "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-NativeFonts", "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SK_USE_DISCARDABLE_SCALEDIMAGECACHE", "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SafeStack", + "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader", "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Shard_12-Coverage", "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All", "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-ASAN", @@ -407,6 +408,7 @@ "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SK_CPU_LIMIT_SSE2", "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SK_CPU_LIMIT_SSE41", "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SK_FORCE_RASTER_PIPELINE_BLITTER", + "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader", "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-TSAN", "Test-Debian9-Clang-GCE-CPU-AVX512-x86_64-Debug-All", "Test-Debian9-Clang-GCE-CPU-AVX512-x86_64-Release-All", diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader.json new file mode 100644 index 0000000000..e5c15d7b11 --- /dev/null +++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader.json @@ -0,0 +1,319 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/skia/infra/bots/assets/clang_linux/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get clang_linux VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/build/out/swiftshader_out" + ], + "infra_step": true, + "name": "makedirs swiftshader_out" + }, + { + "cmd": [ + "cmake", + "[START_DIR]/skia/third_party/externals/swiftshader", + "-GNinja" + ], + "cwd": "[START_DIR]/build/out/swiftshader_out", + "env": { + "BUILDTYPE": "Debug", + "CC": "[START_DIR]/clang_linux/bin/clang", + "CHROME_HEADLESS": "1", + "CXX": "[START_DIR]/clang_linux/bin/clang++", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]:[START_DIR]/cmake_linux/bin", + "SKIA_OUT": "[START_DIR]/build/out" + }, + "name": "swiftshader cmake" + }, + { + "cmd": [ + "ninja", + "-C", + "[START_DIR]/build/out/swiftshader_out", + "libEGL.so", + "libGLESv2.so" + ], + "cwd": "[START_DIR]/build/out/swiftshader_out", + "env": { + "BUILDTYPE": "Debug", + "CC": "[START_DIR]/clang_linux/bin/clang", + "CHROME_HEADLESS": "1", + "CXX": "[START_DIR]/clang_linux/bin/clang++", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]:[START_DIR]/cmake_linux/bin", + "SKIA_OUT": "[START_DIR]/build/out" + }, + "name": "swiftshader ninja" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/skia/bin/fetch-gn" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/build/out" + }, + "infra_step": true, + "name": "fetch-gn" + }, + { + "cmd": [ + "[START_DIR]/skia/bin/gn", + "gen", + "[START_DIR]/build/out/Debug", + "--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\", \"-DDUMMY_clang_linux_version=42\", \"-O1\", \"-DGR_EGL_TRY_GLES3_THEN_GLES2\", \"-I[START_DIR]/skia/third_party/externals/swiftshader/include\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\", \"-L[START_DIR]/build/out/swiftshader_out\"] skia_use_egl=true" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/build/out" + }, + "name": "gn gen" + }, + { + "cmd": [ + "ninja", + "-k", + "0", + "-C", + "[START_DIR]/build/out/Debug" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/build/out" + }, + "name": "ninja" + }, + { + "cmd": [ + "python", + "-u", + "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n", + "[START_DIR]/build/out/swiftshader_out", + "[START_DIR]/[SWARM_OUT_DIR]/out/swiftshader_out" + ], + "infra_step": true, + "name": "copy build products", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import errno@@@", + "@@@STEP_LOG_LINE@python.inline@import glob@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import shutil@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@", + "@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@try:@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@", + "@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@", + "@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@", + "@@@STEP_LOG_LINE@python.inline@ raise@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@", + "@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@", + "@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@", + "@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@", + "@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "results_dir" + ], + "infra_step": true, + "name": "rmtree results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "results_dir" + ], + "infra_step": true, + "name": "makedirs results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "device_results_dir" + ], + "infra_step": true, + "name": "rmtree device_results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "device_results_dir" + ], + "infra_step": true, + "name": "makedirs device_results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/skia/infra/bots/assets/skp/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get skp VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/SKP_VERSION" + ], + "infra_step": true, + "name": "write SKP_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/skia/infra/bots/assets/skimage/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get skimage VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/SK_IMAGE_VERSION" + ], + "infra_step": true, + "name": "write SK_IMAGE_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/skia/infra/bots/assets/svg/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get svg VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/SVG_VERSION" + ], + "infra_step": true, + "name": "write SVG_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::flavor]/resources/symbolize_stack_trace.py", + "[START_DIR]", + "catchsegv", + "[START_DIR]/build/out/Debug/dm", + "--some-flag" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "LD_LIBRARY_PATH": "[START_DIR]/build/out/swiftshader_out", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/build/out" + }, + "name": "symbolized dm" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipe_modules/flavor/examples/full.py b/infra/bots/recipe_modules/flavor/examples/full.py index 19f0df6f2c..cad7c09f98 100644 --- a/infra/bots/recipe_modules/flavor/examples/full.py +++ b/infra/bots/recipe_modules/flavor/examples/full.py @@ -107,6 +107,7 @@ TEST_BUILDERS = [ 'Test-ChromeOS-Clang-SamsungChromebookPlus-GPU-MaliT860-arm-Release-All', 'Test-Debian9-Clang-GCE-CPU-AVX2-universal-devrel-All-Android_SKQP', 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-Coverage', + 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader', 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-TSAN', 'Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Release-All', 'Test-Mac-Clang-MacMini7.1-CPU-AVX-x86_64-Debug-All-ASAN', diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py index 35f20c5703..fa8a4b29d8 100644 --- a/infra/bots/recipe_modules/flavor/gn_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_flavor.py @@ -306,6 +306,9 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): path.append(slave_dir.join('linux_vulkan_sdk', 'bin')) ld_library_path.append(slave_dir.join('linux_vulkan_sdk', 'lib')) + if 'SwiftShader' in extra_tokens: + ld_library_path.append(self.m.vars.skia_out.join('swiftshader_out')) + if 'MSAN' in extra_tokens: # Find the MSAN-built libc++. ld_library_path.append(clang_linux + '/msan') diff --git a/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader.json b/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader.json new file mode 100644 index 0000000000..d613921dad --- /dev/null +++ b/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader.json @@ -0,0 +1,287 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/skia/infra/bots/assets/skp/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get skp VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/SKP_VERSION" + ], + "infra_step": true, + "name": "write SKP_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/skia/infra/bots/assets/skimage/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get skimage VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/SK_IMAGE_VERSION" + ], + "infra_step": true, + "name": "write SK_IMAGE_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/skia/infra/bots/assets/svg/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get svg VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/SVG_VERSION" + ], + "infra_step": true, + "name": "write SVG_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[START_DIR]/test" + ], + "infra_step": true, + "name": "rmtree test" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/test" + ], + "infra_step": true, + "name": "makedirs test" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/tmp" + ], + "infra_step": true, + "name": "makedirs tmp_dir" + }, + { + "cmd": [ + "python", + "-u", + "\nimport contextlib\nimport math\nimport socket\nimport sys\nimport time\nimport urllib2\n\nHASHES_URL = 'https://storage.googleapis.com/skia-infra-gm/hash_files/gold-prod-hashes.txt'\nRETRIES = 5\nTIMEOUT = 60\nWAIT_BASE = 15\n\nsocket.setdefaulttimeout(TIMEOUT)\nfor retry in range(RETRIES):\n try:\n with contextlib.closing(\n urllib2.urlopen(HASHES_URL, timeout=TIMEOUT)) as w:\n hashes = w.read()\n with open(sys.argv[1], 'w') as f:\n f.write(hashes)\n break\n except Exception as e:\n print 'Failed to get uninteresting hashes from %s:' % HASHES_URL\n print e\n if retry == RETRIES:\n raise\n waittime = WAIT_BASE * math.pow(2, retry)\n print 'Retry in %d seconds.' % waittime\n time.sleep(waittime)\n", + "[START_DIR]/tmp/uninteresting_hashes.txt" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/build/out" + }, + "infra_step": true, + "name": "get uninteresting hashes", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import contextlib@@@", + "@@@STEP_LOG_LINE@python.inline@import math@@@", + "@@@STEP_LOG_LINE@python.inline@import socket@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@import time@@@", + "@@@STEP_LOG_LINE@python.inline@import urllib2@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@HASHES_URL = 'https://storage.googleapis.com/skia-infra-gm/hash_files/gold-prod-hashes.txt'@@@", + "@@@STEP_LOG_LINE@python.inline@RETRIES = 5@@@", + "@@@STEP_LOG_LINE@python.inline@TIMEOUT = 60@@@", + "@@@STEP_LOG_LINE@python.inline@WAIT_BASE = 15@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@socket.setdefaulttimeout(TIMEOUT)@@@", + "@@@STEP_LOG_LINE@python.inline@for retry in range(RETRIES):@@@", + "@@@STEP_LOG_LINE@python.inline@ try:@@@", + "@@@STEP_LOG_LINE@python.inline@ with contextlib.closing(@@@", + "@@@STEP_LOG_LINE@python.inline@ urllib2.urlopen(HASHES_URL, timeout=TIMEOUT)) as w:@@@", + "@@@STEP_LOG_LINE@python.inline@ hashes = w.read()@@@", + "@@@STEP_LOG_LINE@python.inline@ with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ f.write(hashes)@@@", + "@@@STEP_LOG_LINE@python.inline@ break@@@", + "@@@STEP_LOG_LINE@python.inline@ except Exception as e:@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Failed to get uninteresting hashes from %s:' % HASHES_URL@@@", + "@@@STEP_LOG_LINE@python.inline@ print e@@@", + "@@@STEP_LOG_LINE@python.inline@ if retry == RETRIES:@@@", + "@@@STEP_LOG_LINE@python.inline@ raise@@@", + "@@@STEP_LOG_LINE@python.inline@ waittime = WAIT_BASE * math.pow(2, retry)@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Retry in %d seconds.' % waittime@@@", + "@@@STEP_LOG_LINE@python.inline@ time.sleep(waittime)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n" + ], + "name": "get swarming bot id", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n" + ], + "name": "get swarming task id", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::flavor]/resources/symbolize_stack_trace.py", + "[START_DIR]", + "catchsegv", + "[START_DIR]/build/out/Release/dm", + "--resourcePath", + "[START_DIR]/skia/resources", + "--skps", + "[START_DIR]/skp", + "--images", + "[START_DIR]/skimage/dm", + "--colorImages", + "[START_DIR]/skimage/colorspace", + "--nameByHash", + "--properties", + "gitHash", + "abc123", + "builder", + "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader", + "buildbucket_build_id", + "123454321", + "swarming_bot_id", + "skia-bot-123", + "swarming_task_id", + "123456", + "--svgs", + "[START_DIR]/svg", + "--key", + "arch", + "x86_64", + "compiler", + "Clang", + "configuration", + "Release", + "cpu_or_gpu", + "CPU", + "cpu_or_gpu_value", + "AVX2", + "extra_config", + "SwiftShader", + "model", + "GCE", + "os", + "Debian9", + "--uninterestingHashesFile", + "[START_DIR]/tmp/uninteresting_hashes.txt", + "--writePath", + "[START_DIR]/[SWARM_OUT_DIR]/dm", + "--dont_write", + "pdf", + "--randomProcessorTest", + "--disableDriverCorrectnessWorkarounds", + "--config", + "gles", + "glesdft", + "--src", + "tests", + "gm", + "image", + "colorImage", + "--nonativeFonts", + "--verbose" + ], + "cwd": "[START_DIR]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "LD_LIBRARY_PATH": "[START_DIR]/build/out/swiftshader_out", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/build/out" + }, + "name": "symbolized dm" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipes/test.py b/infra/bots/recipes/test.py index 811282b57b..849a7a5c09 100644 --- a/infra/bots/recipes/test.py +++ b/infra/bots/recipes/test.py @@ -83,7 +83,11 @@ def dm_flags(api, bot): 'Chromecast' in bot): args.append('--ignoreSigInt') - if api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': + if 'SwiftShader' in api.vars.extra_tokens: + configs.extend(['gles', 'glesdft']) + args.append('--disableDriverCorrectnessWorkarounds') + + elif api.vars.builder_cfg.get('cpu_or_gpu') == 'CPU': args.append('--nogpu') # These are the canonical configs that we would ideally run on all bots. We @@ -966,6 +970,7 @@ TEST_BUILDERS = [ '-SK_USE_DISCARDABLE_SCALEDIMAGECACHE'), ('Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All' '-SK_FORCE_RASTER_PIPELINE_BLITTER'), + 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader', 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-TSAN', 'Test-Debian9-GCC-GCE-CPU-AVX2-x86-Debug-All', 'Test-Debian9-GCC-GCE-CPU-AVX2-x86_64-Debug-All', diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json index 4f15e18e6c..ede69c4d8d 100644 --- a/infra/bots/tasks.json +++ b/infra/bots/tasks.json @@ -2411,6 +2411,12 @@ "Upload-Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SafeStack" ] }, + "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader": { + "priority": 0.8, + "tasks": [ + "Upload-Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader" + ] + }, "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Shard_12-Coverage": { "priority": 0.8, "tasks": [ @@ -2459,6 +2465,12 @@ "Upload-Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SK_FORCE_RASTER_PIPELINE_BLITTER" ] }, + "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader": { + "priority": 0.8, + "tasks": [ + "Upload-Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader" + ] + }, "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-TSAN": { "priority": 0.8, "tasks": [ @@ -44858,6 +44870,114 @@ ], "priority": 0.8 }, + "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader": { + "cipd_packages": [ + { + "name": "infra/tools/luci/kitchen/${platform}", + "path": ".", + "version": "git_revision:206b4474cb712bdad8b7b3f213880cfbf03f120c" + }, + { + "name": "infra/tools/authutil/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:9c63809842a277ce10a86afd51b61c639a665d11" + }, + { + "name": "infra/tools/luci/vpython/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:d0130097bd6364a8d834cb9efd4554c1f6192c82" + }, + { + "name": "skia/bots/skimage", + "path": "skimage", + "version": "version:34" + }, + { + "name": "skia/bots/skp", + "path": "skp", + "version": "version:120" + }, + { + "name": "skia/bots/svg", + "path": "svg", + "version": "version:9" + } + ], + "command": [ + "./kitchen${EXECUTABLE_SUFFIX}", + "cook", + "-checkout-dir", + "recipe_bundle", + "-mode", + "swarming", + "-luci-system-account", + "system", + "-cache-dir", + "cache", + "-temp-dir", + "tmp", + "-known-gerrit-host", + "android.googlesource.com", + "-known-gerrit-host", + "boringssl.googlesource.com", + "-known-gerrit-host", + "chromium.googlesource.com", + "-known-gerrit-host", + "dart.googlesource.com", + "-known-gerrit-host", + "fuchsia.googlesource.com", + "-known-gerrit-host", + "go.googlesource.com", + "-known-gerrit-host", + "llvm.googlesource.com", + "-known-gerrit-host", + "pdfium.googlesource.com", + "-known-gerrit-host", + "skia.googlesource.com", + "-known-gerrit-host", + "webrtc.googlesource.com", + "-output-result-json", + "${ISOLATED_OUTDIR}/build_result_filename", + "-workdir", + ".", + "-recipe", + "test", + "-properties", + "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader\",\"patch_issue\":\"<(ISSUE)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"test\"}", + "-logdog-annotation-url", + "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations" + ], + "dependencies": [ + "Housekeeper-PerCommit-BundleRecipes", + "Build-Debian9-Clang-x86_64-Debug-SwiftShader" + ], + "dimensions": [ + "cpu:x86-64-Haswell_GCE", + "os:Debian-9.4", + "pool:Skia" + ], + "env_prefixes": { + "PATH": [ + "cipd_bin_packages", + "cipd_bin_packages/bin" + ], + "VPYTHON_VIRTUALENV_ROOT": [ + "${cache_dir}/vpython" + ] + }, + "execution_timeout_ns": 14400000000000, + "expiration_ns": 72000000000000, + "extra_tags": { + "log_location": "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations" + }, + "io_timeout_ns": 14400000000000, + "isolate": "test_skia_bundled.isolate", + "max_attempts": 1, + "outputs": [ + "test" + ], + "priority": 0.8 + }, "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-shard_00_12-Coverage": { "cipd_packages": [ { @@ -46915,6 +47035,114 @@ ], "priority": 0.8 }, + "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader": { + "cipd_packages": [ + { + "name": "infra/tools/luci/kitchen/${platform}", + "path": ".", + "version": "git_revision:206b4474cb712bdad8b7b3f213880cfbf03f120c" + }, + { + "name": "infra/tools/authutil/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:9c63809842a277ce10a86afd51b61c639a665d11" + }, + { + "name": "infra/tools/luci/vpython/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:d0130097bd6364a8d834cb9efd4554c1f6192c82" + }, + { + "name": "skia/bots/skimage", + "path": "skimage", + "version": "version:34" + }, + { + "name": "skia/bots/skp", + "path": "skp", + "version": "version:120" + }, + { + "name": "skia/bots/svg", + "path": "svg", + "version": "version:9" + } + ], + "command": [ + "./kitchen${EXECUTABLE_SUFFIX}", + "cook", + "-checkout-dir", + "recipe_bundle", + "-mode", + "swarming", + "-luci-system-account", + "system", + "-cache-dir", + "cache", + "-temp-dir", + "tmp", + "-known-gerrit-host", + "android.googlesource.com", + "-known-gerrit-host", + "boringssl.googlesource.com", + "-known-gerrit-host", + "chromium.googlesource.com", + "-known-gerrit-host", + "dart.googlesource.com", + "-known-gerrit-host", + "fuchsia.googlesource.com", + "-known-gerrit-host", + "go.googlesource.com", + "-known-gerrit-host", + "llvm.googlesource.com", + "-known-gerrit-host", + "pdfium.googlesource.com", + "-known-gerrit-host", + "skia.googlesource.com", + "-known-gerrit-host", + "webrtc.googlesource.com", + "-output-result-json", + "${ISOLATED_OUTDIR}/build_result_filename", + "-workdir", + ".", + "-recipe", + "test", + "-properties", + "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader\",\"patch_issue\":\"<(ISSUE)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"test\"}", + "-logdog-annotation-url", + "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations" + ], + "dependencies": [ + "Housekeeper-PerCommit-BundleRecipes", + "Build-Debian9-Clang-x86_64-Release-SwiftShader" + ], + "dimensions": [ + "cpu:x86-64-Haswell_GCE", + "os:Debian-9.4", + "pool:Skia" + ], + "env_prefixes": { + "PATH": [ + "cipd_bin_packages", + "cipd_bin_packages/bin" + ], + "VPYTHON_VIRTUALENV_ROOT": [ + "${cache_dir}/vpython" + ] + }, + "execution_timeout_ns": 14400000000000, + "expiration_ns": 72000000000000, + "extra_tags": { + "log_location": "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations" + }, + "io_timeout_ns": 14400000000000, + "isolate": "test_skia_bundled.isolate", + "max_attempts": 1, + "outputs": [ + "test" + ], + "priority": 0.8 + }, "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-TSAN": { "cipd_packages": [ { @@ -80650,6 +80878,99 @@ "priority": 0.8, "service_account": "skia-external-gm-uploader@skia-swarming-bots.iam.gserviceaccount.com" }, + "Upload-Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader": { + "cipd_packages": [ + { + "name": "infra/tools/luci/kitchen/${platform}", + "path": ".", + "version": "git_revision:206b4474cb712bdad8b7b3f213880cfbf03f120c" + }, + { + "name": "infra/tools/authutil/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:9c63809842a277ce10a86afd51b61c639a665d11" + }, + { + "name": "infra/tools/luci/vpython/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:d0130097bd6364a8d834cb9efd4554c1f6192c82" + }, + { + "name": "infra/gsutil", + "path": "cipd_bin_packages", + "version": "version:4.28" + } + ], + "command": [ + "./kitchen${EXECUTABLE_SUFFIX}", + "cook", + "-checkout-dir", + "recipe_bundle", + "-mode", + "swarming", + "-luci-system-account", + "system", + "-cache-dir", + "cache", + "-temp-dir", + "tmp", + "-known-gerrit-host", + "android.googlesource.com", + "-known-gerrit-host", + "boringssl.googlesource.com", + "-known-gerrit-host", + "chromium.googlesource.com", + "-known-gerrit-host", + "dart.googlesource.com", + "-known-gerrit-host", + "fuchsia.googlesource.com", + "-known-gerrit-host", + "go.googlesource.com", + "-known-gerrit-host", + "llvm.googlesource.com", + "-known-gerrit-host", + "pdfium.googlesource.com", + "-known-gerrit-host", + "skia.googlesource.com", + "-known-gerrit-host", + "webrtc.googlesource.com", + "-output-result-json", + "${ISOLATED_OUTDIR}/build_result_filename", + "-workdir", + ".", + "-recipe", + "upload_dm_results", + "-properties", + "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader\",\"gs_bucket\":\"skia-infra-gm\",\"patch_issue\":\"<(ISSUE)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"output_ignored\"}", + "-logdog-annotation-url", + "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations" + ], + "dependencies": [ + "Housekeeper-PerCommit-BundleRecipes", + "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-SwiftShader" + ], + "dimensions": [ + "cpu:x86-64-Haswell_GCE", + "gpu:none", + "os:Debian-9.4", + "pool:Skia" + ], + "env_prefixes": { + "PATH": [ + "cipd_bin_packages", + "cipd_bin_packages/bin" + ], + "VPYTHON_VIRTUALENV_ROOT": [ + "${cache_dir}/vpython" + ] + }, + "extra_tags": { + "log_location": "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations" + }, + "isolate": "swarm_recipe.isolate", + "priority": 0.8, + "service_account": "skia-external-gm-uploader@skia-swarming-bots.iam.gserviceaccount.com" + }, "Upload-Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-Shard_12-Coverage": { "cipd_packages": [ { @@ -81348,6 +81669,99 @@ "priority": 0.8, "service_account": "skia-external-gm-uploader@skia-swarming-bots.iam.gserviceaccount.com" }, + "Upload-Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader": { + "cipd_packages": [ + { + "name": "infra/tools/luci/kitchen/${platform}", + "path": ".", + "version": "git_revision:206b4474cb712bdad8b7b3f213880cfbf03f120c" + }, + { + "name": "infra/tools/authutil/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:9c63809842a277ce10a86afd51b61c639a665d11" + }, + { + "name": "infra/tools/luci/vpython/${platform}", + "path": "cipd_bin_packages", + "version": "git_revision:d0130097bd6364a8d834cb9efd4554c1f6192c82" + }, + { + "name": "infra/gsutil", + "path": "cipd_bin_packages", + "version": "version:4.28" + } + ], + "command": [ + "./kitchen${EXECUTABLE_SUFFIX}", + "cook", + "-checkout-dir", + "recipe_bundle", + "-mode", + "swarming", + "-luci-system-account", + "system", + "-cache-dir", + "cache", + "-temp-dir", + "tmp", + "-known-gerrit-host", + "android.googlesource.com", + "-known-gerrit-host", + "boringssl.googlesource.com", + "-known-gerrit-host", + "chromium.googlesource.com", + "-known-gerrit-host", + "dart.googlesource.com", + "-known-gerrit-host", + "fuchsia.googlesource.com", + "-known-gerrit-host", + "go.googlesource.com", + "-known-gerrit-host", + "llvm.googlesource.com", + "-known-gerrit-host", + "pdfium.googlesource.com", + "-known-gerrit-host", + "skia.googlesource.com", + "-known-gerrit-host", + "webrtc.googlesource.com", + "-output-result-json", + "${ISOLATED_OUTDIR}/build_result_filename", + "-workdir", + ".", + "-recipe", + "upload_dm_results", + "-properties", + "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildbucket_build_id\":\"<(BUILDBUCKET_BUILD_ID)\",\"buildername\":\"Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader\",\"gs_bucket\":\"skia-infra-gm\",\"patch_issue\":\"<(ISSUE)\",\"patch_repo\":\"<(PATCH_REPO)\",\"patch_set\":\"<(PATCHSET)\",\"patch_storage\":\"<(PATCH_STORAGE)\",\"repository\":\"<(REPO)\",\"revision\":\"<(REVISION)\",\"swarm_out_dir\":\"output_ignored\"}", + "-logdog-annotation-url", + "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations" + ], + "dependencies": [ + "Housekeeper-PerCommit-BundleRecipes", + "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-SwiftShader" + ], + "dimensions": [ + "cpu:x86-64-Haswell_GCE", + "gpu:none", + "os:Debian-9.4", + "pool:Skia" + ], + "env_prefixes": { + "PATH": [ + "cipd_bin_packages", + "cipd_bin_packages/bin" + ], + "VPYTHON_VIRTUALENV_ROOT": [ + "${cache_dir}/vpython" + ] + }, + "extra_tags": { + "log_location": "logdog://logs.chromium.org/skia/<(TASK_ID)/+/annotations" + }, + "isolate": "swarm_recipe.isolate", + "priority": 0.8, + "service_account": "skia-external-gm-uploader@skia-swarming-bots.iam.gserviceaccount.com" + }, "Upload-Test-Debian9-Clang-GCE-CPU-AVX512-x86_64-Debug-All": { "cipd_packages": [ { -- cgit v1.2.3