diff options
| author |  rmistry <rmistry@google.com> | 2016-08-24 05:07:06 -0700 |
|---|---|---|
| committer |  Commit bot <commit-bot@chromium.org> | 2016-08-24 05:07:06 -0700 |
| commit | 3f1c9c0a7227d4ea45c39eb1e596538bb483f65a (patch) | |
| tree | 55208de5c2469cc70cedce760ae3a753d75f0b66 /infra | |
| parent | 6a88206b2ece8b64534c87fca156e5fe161f884e (diff) | |
Apply gerrit ref if it is a Gerrit patch.
Also:
* Pass through a new property 'patch_storage' to DM/Nanobench/Coverage. This will be used by the different frameworks to figure out if it is Rietveld or Gerrit issue.
* Calculate issue and patchset for Gerrit patches similar to Rietveld.
BUG=skia:5627
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2263323002
Review-Url: https://codereview.chromium.org/2263323002
Diffstat (limited to 'infra')
16 files changed, 2028 insertions, 16 deletions
diff --git a/infra/bots/recipe_modules/core/__init__.py b/infra/bots/recipe_modules/core/__init__.py index df4078b271..8fe6053715 100644 --- a/infra/bots/recipe_modules/core/__init__.py +++ b/infra/bots/recipe_modules/core/__init__.py @@ -4,6 +4,7 @@ DEPS = [ 'build/file', + 'depot_tools/bot_update', 'depot_tools/gclient', 'depot_tools/tryserver', 'flavor', diff --git a/infra/bots/recipe_modules/core/api.py b/infra/bots/recipe_modules/core/api.py index e65073cb36..bc96d99a85 100644 --- a/infra/bots/recipe_modules/core/api.py +++ b/infra/bots/recipe_modules/core/api.py @@ -140,6 +140,9 @@ class SkiaApi(recipe_api.RecipeApi): self.m.vars.got_revision = ( update_step.presentation.properties['got_revision']) self.m.tryserver.maybe_apply_issue() + if self.m.properties.get('patch_storage') == 'gerrit': + self.m.bot_update.apply_gerrit_ref( + root=str(self.m.vars.checkout_root.join('skia'))) if self.m.vars.need_chromium_checkout: self.m.gclient.runhooks(cwd=self.m.vars.checkout_root, diff --git a/infra/bots/recipe_modules/vars/api.py b/infra/bots/recipe_modules/vars/api.py index e2d0ed99c0..5418e26022 100644 --- a/infra/bots/recipe_modules/vars/api.py +++ b/infra/bots/recipe_modules/vars/api.py @@ -110,13 +110,16 @@ class SkiaVarsApi(recipe_api.RecipeApi): self.default_env.update({'SKIA_OUT': self.skia_out, 'BUILDTYPE': self.configuration}) self.is_trybot = self.builder_cfg['is_trybot'] + self.patch_storage = self.m.properties.get('patch_storage', '') self.issue = None self.patchset = None - self.rietveld = None if self.is_trybot: - self.issue = self.m.properties['issue'] - self.patchset = self.m.properties['patchset'] - self.rietveld = self.m.properties['rietveld'] + if self.patch_storage == 'gerrit': + self.issue = self.m.properties['event.change.number'] + self.patchset = self.m.properties['event.patchSet.ref'].split('/')[-1] + else: + self.issue = self.m.properties['issue'] + self.patchset = self.m.properties['patchset'] self.dm_dir = self.m.path.join( self.swarming_out_dir, 'dm') self.perf_data_dir = self.m.path.join(self.swarming_out_dir, diff --git a/infra/bots/recipes/swarm_compile.expected/recipe_with_gerrit_patch.json b/infra/bots/recipes/swarm_compile.expected/recipe_with_gerrit_patch.json new file mode 100644 index 0000000000..4fb085d331 --- /dev/null +++ b/infra/bots/recipes/swarm_compile.expected/recipe_with_gerrit_patch.json @@ -0,0 +1,206 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_C:\\_B_WORK]", + "511" + ], + "name": "makedirs checkout_path", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py", + "config", + "--spec", + "cache_dir = '[CUSTOM_C:\\\\_B_CACHE]'\nsolutions = [{'deps_file': 'DEPS', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]\ntarget_os = ['llvm']" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug-Trybot" + }, + "name": "gclient setup" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]\\gclient.py", + "sync", + "--nohooks", + "--force", + "--verbose", + "--delete_unversioned_trees", + "--revision", + "skia@abc123", + "--output-json", + "/path/to/tmp/json" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug-Trybot" + }, + "name": "gclient sync", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"solutions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia/\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": 164710@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@164710@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]\\resources\\apply_gerrit.py", + "--gerrit_repo", + "skia", + "--gerrit_ref", + "refs/changes/00/2100/2", + "--root", + "[CUSTOM_C:\\_B_WORK]\\skia", + "--gerrit_rebase_patch_ref" + ], + "env": { + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "apply_gerrit" + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]\\tmp", + "511" + ], + "name": "makedirs tmp_dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CUSTOM_C:\\_B_WORK]\\skia\\infra\\bots\\bootstrap_win_toolchain_json.py", + "--win_toolchain_json", + "[SLAVE_BUILD]\\src\\build\\win_toolchain.json", + "--depot_tools_parent_dir", + "[SLAVE_BUILD]" + ], + "name": "bootstrap win toolchain" + }, + { + "cmd": [ + "python", + "make.py", + "most" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]\\skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "CHROME_PATH": "[SLAVE_BUILD]\\src", + "GYP_DEFINES": "qt_sdk=C:/Qt/4.8.5/ skia_arch_type=x86 skia_warnings_as_errors=1 skia_win_debuggers_path=c:/DbgHelp skia_win_ltcg=0", + "PATH": "%(PATH)s;RECIPE_PACKAGE_REPO[depot_tools];RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug-Trybot" + }, + "name": "build most" + }, + { + "cmd": [ + "python", + "-u", + "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n", + "[CUSTOM_C:\\_B_WORK]\\skia\\out\\Build-Win-MSVC-x86-Debug-Trybot\\Debug", + "[CUSTOM_[SWARM_OUT_DIR]]\\out\\Debug" + ], + "name": "copy build products", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import errno@@@", + "@@@STEP_LOG_LINE@python.inline@import glob@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import shutil@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@", + "@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['dm', 'dm.exe', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'lib/*.so', 'iOSShell.app', 'iOSShell.ipa', 'visualbench', 'visualbench.exe', 'vulkan-1.dll']@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@try:@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@", + "@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@", + "@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@", + "@@@STEP_LOG_LINE@python.inline@ raise@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@", + "@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@", + "@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@", + "@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@", + "@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "import psutil\nfor p in psutil.process_iter():\n try:\n if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):\n p.kill()\n except psutil._error.AccessDenied:\n pass\n" + ], + "name": "cleanup", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import psutil@@@", + "@@@STEP_LOG_LINE@python.inline@for p in psutil.process_iter():@@@", + "@@@STEP_LOG_LINE@python.inline@ try:@@@", + "@@@STEP_LOG_LINE@python.inline@ if p.name in ('mspdbsrv.exe', 'vctip.exe', 'cl.exe', 'link.exe'):@@@", + "@@@STEP_LOG_LINE@python.inline@ p.kill()@@@", + "@@@STEP_LOG_LINE@python.inline@ except psutil._error.AccessDenied:@@@", + "@@@STEP_LOG_LINE@python.inline@ pass@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipes/swarm_compile.py b/infra/bots/recipes/swarm_compile.py index cdf4739f44..5ca28edfe5 100644 --- a/infra/bots/recipes/swarm_compile.py +++ b/infra/bots/recipes/swarm_compile.py @@ -107,7 +107,7 @@ def get_gyp_defines(builder_dict): werr = False elif 'Fast' in builder_dict.get('extra_config', ''): # See https://bugs.chromium.org/p/skia/issues/detail?id=5257 - werr = False + werr = False else: werr = True gyp_defs['skia_warnings_as_errors'] = str(int(werr)) # True/False -> '1'/'0' @@ -309,3 +309,23 @@ def GenTests(api): ) + api.platform('win', 64) ) + + gerrit_kwargs = { + 'patch_storage': 'gerrit', + 'repository': 'skia', + 'event.patchSet.ref': 'refs/changes/00/2100/2', + 'event.change.number': '2100', + } + yield ( + api.test('recipe_with_gerrit_patch') + + api.properties( + buildername=buildername + '-Trybot', + mastername=mastername, + slavename=slavename, + buildnumber=5, + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]', + revision='abc123', + **gerrit_kwargs) + + api.platform('win', 64) + ) diff --git a/infra/bots/recipes/swarm_perf.expected/Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot.json b/infra/bots/recipes/swarm_perf.expected/Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot.json index 4bc4304d70..4f27376436 100644 --- a/infra/bots/recipes/swarm_perf.expected/Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot.json +++ b/infra/bots/recipes/swarm_perf.expected/Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot.json @@ -167,6 +167,8 @@ "500", "patchset", "1", + "patch_storage", + "", "--key", "arch", "x86_64", diff --git a/infra/bots/recipes/swarm_perf.expected/big_issue_number.json b/infra/bots/recipes/swarm_perf.expected/big_issue_number.json index 5ebf500484..ce071813e8 100644 --- a/infra/bots/recipes/swarm_perf.expected/big_issue_number.json +++ b/infra/bots/recipes/swarm_perf.expected/big_issue_number.json @@ -167,6 +167,8 @@ "2147533002", "patchset", "1", + "patch_storage", + "", "--key", "arch", "x86_64", diff --git a/infra/bots/recipes/swarm_perf.expected/recipe_with_gerrit_patch.json b/infra/bots/recipes/swarm_perf.expected/recipe_with_gerrit_patch.json new file mode 100644 index 0000000000..887fd4f6c3 --- /dev/null +++ b/infra/bots/recipes/swarm_perf.expected/recipe_with_gerrit_patch.json @@ -0,0 +1,361 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/tmp", + "511" + ], + "name": "makedirs tmp_dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[SLAVE_BUILD]/skia/infra/bots/assets/skp/VERSION", + "/path/to/tmp/" + ], + "name": "Get downloaded SKP VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "42", + "[SLAVE_BUILD]/tmp/SKP_VERSION" + ], + "name": "write SKP_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[SLAVE_BUILD]/skia/infra/bots/assets/skimage/VERSION", + "/path/to/tmp/" + ], + "name": "Get downloaded skimage VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "42", + "[SLAVE_BUILD]/tmp/SK_IMAGE_VERSION" + ], + "name": "write SK_IMAGE_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[SLAVE_BUILD]/skia/infra/bots/assets/svg/VERSION", + "/path/to/tmp/" + ], + "name": "Get downloaded SVG VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "42", + "[SLAVE_BUILD]/tmp/SVG_VERSION" + ], + "name": "write SVG_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", + "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind-Trybot/data" + ], + "env": { + "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" + }, + "name": "rmtree data", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", + "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", + "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind-Trybot/data", + "511" + ], + "name": "makedirs data", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "valgrind", + "--gen-suppressions=all", + "--leak-check=full", + "--track-origins=yes", + "--error-exitcode=1", + "--num-callers=40", + "--suppressions=[SLAVE_BUILD]/skia/tools/valgrind.supp", + "[SLAVE_BUILD]/out/Release/nanobench", + "--undefok", + "-i", + "[SLAVE_BUILD]/skia/resources", + "--skps", + "[SLAVE_BUILD]/skp", + "--images", + "[SLAVE_BUILD]/skimage/nanobench", + "--nocpu", + "--pre_log", + "--images", + "--gpuStatsDump", + "true", + "--scales", + "1.0", + "1.1", + "--config", + "565", + "8888", + "gpu", + "nonrendering", + "angle", + "hwui", + "f16", + "srgb", + "msaa16", + "nvpr16", + "nvprdit16", + "--loops", + "1", + "--samples", + "1", + "--keepAlive", + "true", + "--match", + "~interlaced1.png", + "~interlaced2.png", + "~interlaced3.png", + "~inc0.gif", + "~inc1.gif", + "~incInterlaced.gif", + "~inc0.jpg", + "~incGray.jpg", + "~inc0.wbmp", + "~inc1.wbmp", + "~inc0.webp", + "~inc1.webp", + "~inc0.ico", + "~inc1.ico", + "~inc0.png", + "~inc1.png", + "~inc2.png", + "~inc12.png", + "~inc13.png", + "~inc14.png", + "~inc0.webp", + "~inc1.webp", + "--outResultsFile", + "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind-Trybot/data/nanobench_abc123.json", + "--properties", + "gitHash", + "abc123", + "build_number", + "5", + "issue", + "2100", + "patchset", + "2", + "patch_storage", + "gerrit", + "--key", + "arch", + "x86_64", + "compiler", + "GCC", + "cpu_or_gpu", + "GPU", + "cpu_or_gpu_value", + "GTX550Ti", + "extra_config", + "Valgrind", + "model", + "ShuttleA", + "os", + "Ubuntu" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "SKIA_OUT": "[SLAVE_BUILD]/out" + }, + "name": "nanobench" + }, + { + "cmd": [ + "valgrind", + "--gen-suppressions=all", + "--leak-check=full", + "--track-origins=yes", + "--error-exitcode=1", + "--num-callers=40", + "--suppressions=[SLAVE_BUILD]/skia/tools/valgrind.supp", + "[SLAVE_BUILD]/out/Release/nanobench", + "--undefok", + "-i", + "[SLAVE_BUILD]/skia/resources", + "--skps", + "[SLAVE_BUILD]/skp", + "--images", + "[SLAVE_BUILD]/skimage/nanobench", + "--nocpu", + "--pre_log", + "--images", + "--gpuStatsDump", + "true", + "--scales", + "1.0", + "1.1", + "--config", + "565", + "8888", + "gpu", + "nonrendering", + "angle", + "hwui", + "f16", + "srgb", + "msaa16", + "nvpr16", + "nvprdit16", + "--loops", + "1", + "--samples", + "1", + "--keepAlive", + "true", + "--match", + "~interlaced1.png", + "~interlaced2.png", + "~interlaced3.png", + "~inc0.gif", + "~inc1.gif", + "~incInterlaced.gif", + "~inc0.jpg", + "~incGray.jpg", + "~inc0.wbmp", + "~inc1.wbmp", + "~inc0.webp", + "~inc1.webp", + "~inc0.ico", + "~inc1.ico", + "~inc0.png", + "~inc1.png", + "~inc2.png", + "~inc12.png", + "~inc13.png", + "~inc14.png", + "~inc0.webp", + "~inc1.webp", + "--outResultsFile", + "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind-Trybot/data/nanobench_abc123.json", + "--properties", + "gitHash", + "abc123", + "build_number", + "5", + "issue", + "2100", + "patchset", + "2", + "patch_storage", + "gerrit", + "--key", + "arch", + "x86_64", + "compiler", + "GCC", + "cpu_or_gpu", + "GPU", + "cpu_or_gpu_value", + "GTX550Ti", + "extra_config", + "Valgrind", + "model", + "ShuttleA", + "os", + "Ubuntu", + "--abandonGpuContext", + "--nocpu" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "SKIA_OUT": "[SLAVE_BUILD]/out" + }, + "name": "nanobench --abandonGpuContext" + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_[SWARM_OUT_DIR]]/perfdata/Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind-Trybot/data", + "511" + ], + "name": "makedirs perf_dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipes/swarm_perf.py b/infra/bots/recipes/swarm_perf.py index 60ebaee88b..0701809a90 100644 --- a/infra/bots/recipes/swarm_perf.py +++ b/infra/bots/recipes/swarm_perf.py @@ -173,6 +173,7 @@ def perf_steps(api): properties.extend([ 'issue', api.vars.issue, 'patchset', api.vars.patchset, + 'patch_storage', api.vars.patch_storage, ]) target = 'nanobench' @@ -327,3 +328,23 @@ def GenTests(api): ) + api.platform('win', 64) ) + + gerrit_kwargs = { + 'patch_storage': 'gerrit', + 'repository': 'skia', + 'event.patchSet.ref': 'refs/changes/00/2100/2', + 'event.change.number': '2100', + } + yield ( + api.test('recipe_with_gerrit_patch') + + api.properties( + buildername='Perf-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-' + + 'Valgrind-Trybot', + mastername='client.skia', + slavename='skiabot-linux-swarm-000', + buildnumber=5, + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]', + revision='abc123', + **gerrit_kwargs) + ) diff --git a/infra/bots/recipes/swarm_test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json b/infra/bots/recipes/swarm_test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json index e6065a204b..fb9ae8de6c 100644 --- a/infra/bots/recipes/swarm_test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json +++ b/infra/bots/recipes/swarm_test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage-Trybot.json @@ -147,6 +147,8 @@ "500", "patchset", "1", + "patch_storage", + "", "--svgs", "[SLAVE_BUILD]/svg", "--key", @@ -483,7 +485,9 @@ "issue", "500", "patchset", - "1" + "1", + "patch_storage", + "" ], "cwd": "[SLAVE_BUILD]/skia", "env": { diff --git a/infra/bots/recipes/swarm_test.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release-Trybot.json b/infra/bots/recipes/swarm_test.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release-Trybot.json index 02b9741274..96597aa308 100644 --- a/infra/bots/recipes/swarm_test.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release-Trybot.json +++ b/infra/bots/recipes/swarm_test.expected/Test-Win8-MSVC-ShuttleB-CPU-AVX2-x86_64-Release-Trybot.json @@ -199,6 +199,8 @@ "500", "patchset", "1", + "patch_storage", + "", "--svgs", "[SLAVE_BUILD]\\svg", "--key", diff --git a/infra/bots/recipes/swarm_test.expected/big_issue_number.json b/infra/bots/recipes/swarm_test.expected/big_issue_number.json index 60ead9cc37..4662fa30eb 100644 --- a/infra/bots/recipes/swarm_test.expected/big_issue_number.json +++ b/infra/bots/recipes/swarm_test.expected/big_issue_number.json @@ -199,6 +199,8 @@ "2147533002", "patchset", "1", + "patch_storage", + "", "--svgs", "[SLAVE_BUILD]\\svg", "--key", diff --git a/infra/bots/recipes/swarm_test.expected/recipe_with_gerrit_patch.json b/infra/bots/recipes/swarm_test.expected/recipe_with_gerrit_patch.json new file mode 100644 index 0000000000..35ccfbf362 --- /dev/null +++ b/infra/bots/recipes/swarm_test.expected/recipe_with_gerrit_patch.json @@ -0,0 +1,509 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/tmp", + "511" + ], + "name": "makedirs tmp_dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[SLAVE_BUILD]/skia/infra/bots/assets/skp/VERSION", + "/path/to/tmp/" + ], + "name": "Get downloaded SKP VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "42", + "[SLAVE_BUILD]/tmp/SKP_VERSION" + ], + "name": "write SKP_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[SLAVE_BUILD]/skia/infra/bots/assets/skimage/VERSION", + "/path/to/tmp/" + ], + "name": "Get downloaded skimage VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "42", + "[SLAVE_BUILD]/tmp/SK_IMAGE_VERSION" + ], + "name": "write SK_IMAGE_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[SLAVE_BUILD]/skia/infra/bots/assets/svg/VERSION", + "/path/to/tmp/" + ], + "name": "Get downloaded SVG VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "42", + "[SLAVE_BUILD]/tmp/SVG_VERSION" + ], + "name": "write SVG_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", + "[CUSTOM_[SWARM_OUT_DIR]]/dm" + ], + "env": { + "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" + }, + "name": "rmtree dm", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", + "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", + "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[CUSTOM_[SWARM_OUT_DIR]]/dm", + "511" + ], + "name": "makedirs dm", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport contextlib\nimport math\nimport socket\nimport sys\nimport time\nimport urllib2\n\nHASHES_URL = 'https://gold.skia.org/_/hashes'\nRETRIES = 5\nTIMEOUT = 60\nWAIT_BASE = 15\n\nsocket.setdefaulttimeout(TIMEOUT)\nfor retry in range(RETRIES):\n try:\n with contextlib.closing(\n urllib2.urlopen(HASHES_URL, timeout=TIMEOUT)) as w:\n hashes = w.read()\n with open(sys.argv[1], 'w') as f:\n f.write(hashes)\n break\n except Exception as e:\n print 'Failed to get uninteresting hashes from %s:' % HASHES_URL\n print e\n if retry == RETRIES:\n raise\n waittime = WAIT_BASE * math.pow(2, retry)\n print 'Retry in %d seconds.' % waittime\n time.sleep(waittime)\n", + "[SLAVE_BUILD]/tmp/uninteresting_hashes.txt" + ], + "cwd": "[SLAVE_BUILD]/skia", + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "SKIA_OUT": "[SLAVE_BUILD]/out" + }, + "name": "get uninteresting hashes", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import contextlib@@@", + "@@@STEP_LOG_LINE@python.inline@import math@@@", + "@@@STEP_LOG_LINE@python.inline@import socket@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@import time@@@", + "@@@STEP_LOG_LINE@python.inline@import urllib2@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@HASHES_URL = 'https://gold.skia.org/_/hashes'@@@", + "@@@STEP_LOG_LINE@python.inline@RETRIES = 5@@@", + "@@@STEP_LOG_LINE@python.inline@TIMEOUT = 60@@@", + "@@@STEP_LOG_LINE@python.inline@WAIT_BASE = 15@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@socket.setdefaulttimeout(TIMEOUT)@@@", + "@@@STEP_LOG_LINE@python.inline@for retry in range(RETRIES):@@@", + "@@@STEP_LOG_LINE@python.inline@ try:@@@", + "@@@STEP_LOG_LINE@python.inline@ with contextlib.closing(@@@", + "@@@STEP_LOG_LINE@python.inline@ urllib2.urlopen(HASHES_URL, timeout=TIMEOUT)) as w:@@@", + "@@@STEP_LOG_LINE@python.inline@ hashes = w.read()@@@", + "@@@STEP_LOG_LINE@python.inline@ with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ f.write(hashes)@@@", + "@@@STEP_LOG_LINE@python.inline@ break@@@", + "@@@STEP_LOG_LINE@python.inline@ except Exception as e:@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Failed to get uninteresting hashes from %s:' % HASHES_URL@@@", + "@@@STEP_LOG_LINE@python.inline@ print e@@@", + "@@@STEP_LOG_LINE@python.inline@ if retry == RETRIES:@@@", + "@@@STEP_LOG_LINE@python.inline@ raise@@@", + "@@@STEP_LOG_LINE@python.inline@ waittime = WAIT_BASE * math.pow(2, retry)@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Retry in %d seconds.' % waittime@@@", + "@@@STEP_LOG_LINE@python.inline@ time.sleep(waittime)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "[SLAVE_BUILD]/out/Debug/dm", + "--undefok", + "--resourcePath", + "[SLAVE_BUILD]/skia/resources", + "--skps", + "[SLAVE_BUILD]/skp", + "--images", + "[SLAVE_BUILD]/skimage/dm", + "--colorImages", + "[SLAVE_BUILD]/skimage/colorspace", + "--nameByHash", + "--properties", + "gitHash", + "abc123", + "master", + "client.skia", + "builder", + "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug-Trybot", + "build_number", + "5", + "issue", + "2100", + "patchset", + "2", + "patch_storage", + "gerrit", + "--svgs", + "[SLAVE_BUILD]/svg", + "--key", + "arch", + "x86", + "compiler", + "GCC", + "configuration", + "Debug", + "cpu_or_gpu", + "CPU", + "cpu_or_gpu_value", + "AVX2", + "model", + "GCE", + "os", + "Ubuntu", + "--writePath", + "[CUSTOM_[SWARM_OUT_DIR]]/dm", + "--nogpu", + "--threads", + "4", + "--config", + "565", + "8888", + "gpu", + "gpusrgb", + "pdf", + "msaa16", + "f16", + "srgb", + "sp-8888", + "2ndpic-8888", + "lite-8888", + "serialize-8888", + "tiles_rt-8888", + "pic-8888", + "--src", + "tests", + "gm", + "image", + "colorImage", + "svg", + "--blacklist", + "f16", + "_", + "_", + "dstreadshuffle", + "f16", + "image", + "_", + "_", + "srgb", + "image", + "_", + "_", + "gpusrgb", + "image", + "_", + "_", + "serialize-8888", + "gm", + "_", + "bleed_image", + "serialize-8888", + "gm", + "_", + "c_gms", + "serialize-8888", + "gm", + "_", + "colortype", + "serialize-8888", + "gm", + "_", + "colortype_xfermodes", + "serialize-8888", + "gm", + "_", + "drawfilter", + "serialize-8888", + "gm", + "_", + "fontmgr_bounds_0.75_0", + "serialize-8888", + "gm", + "_", + "fontmgr_bounds_1_-0.25", + "serialize-8888", + "gm", + "_", + "fontmgr_bounds", + "serialize-8888", + "gm", + "_", + "fontmgr_match", + "serialize-8888", + "gm", + "_", + "fontmgr_iter", + "serialize-8888", + "gm", + "_", + "bitmapfilters", + "serialize-8888", + "gm", + "_", + "bitmapshaders", + "serialize-8888", + "gm", + "_", + "bleed", + "serialize-8888", + "gm", + "_", + "bleed_alpha_bmp", + "serialize-8888", + "gm", + "_", + "bleed_alpha_bmp_shader", + "serialize-8888", + "gm", + "_", + "convex_poly_clip", + "serialize-8888", + "gm", + "_", + "extractalpha", + "serialize-8888", + "gm", + "_", + "filterbitmap_checkerboard_32_32_g8", + "serialize-8888", + "gm", + "_", + "filterbitmap_image_mandrill_64", + "serialize-8888", + "gm", + "_", + "shadows", + "serialize-8888", + "gm", + "_", + "simpleaaclip_aaclip", + "serialize-8888", + "gm", + "_", + "composeshader_bitmap", + "serialize-8888", + "gm", + "_", + "scaled_tilemodes_npot", + "serialize-8888", + "gm", + "_", + "scaled_tilemodes", + "serialize-8888", + "gm", + "_", + "bleed_alpha_image", + "serialize-8888", + "gm", + "_", + "bleed_alpha_image_shader", + "sp-8888", + "gm", + "_", + "drawfilter", + "pic-8888", + "gm", + "_", + "drawfilter", + "2ndpic-8888", + "gm", + "_", + "drawfilter", + "lite-8888", + "gm", + "_", + "drawfilter", + "sp-8888", + "gm", + "_", + "image-cacherator-from-picture", + "pic-8888", + "gm", + "_", + "image-cacherator-from-picture", + "2ndpic-8888", + "gm", + "_", + "image-cacherator-from-picture", + "serialize-8888", + "gm", + "_", + "image-cacherator-from-picture", + "sp-8888", + "gm", + "_", + "image-cacherator-from-raster", + "pic-8888", + "gm", + "_", + "image-cacherator-from-raster", + "2ndpic-8888", + "gm", + "_", + "image-cacherator-from-raster", + "serialize-8888", + "gm", + "_", + "image-cacherator-from-raster", + "sp-8888", + "gm", + "_", + "image-cacherator-from-ctable", + "pic-8888", + "gm", + "_", + "image-cacherator-from-ctable", + "2ndpic-8888", + "gm", + "_", + "image-cacherator-from-ctable", + "serialize-8888", + "gm", + "_", + "image-cacherator-from-ctable", + "sp-8888", + "svg", + "_", + "car.svg", + "pic-8888", + "svg", + "_", + "car.svg", + "2ndpic-8888", + "svg", + "_", + "car.svg", + "serialize-8888", + "svg", + "_", + "car.svg", + "sp-8888", + "svg", + "_", + "gallardo.svg", + "pic-8888", + "svg", + "_", + "gallardo.svg", + "2ndpic-8888", + "svg", + "_", + "gallardo.svg", + "serialize-8888", + "svg", + "_", + "gallardo.svg", + "sp-8888", + "svg", + "_", + "rg1024_green_grapes.svg", + "pic-8888", + "svg", + "_", + "rg1024_green_grapes.svg", + "2ndpic-8888", + "svg", + "_", + "rg1024_green_grapes.svg", + "serialize-8888", + "svg", + "_", + "rg1024_green_grapes.svg", + "sp-8888", + "svg", + "_", + "Seal_of_Kansas.svg", + "pic-8888", + "svg", + "_", + "Seal_of_Kansas.svg", + "2ndpic-8888", + "svg", + "_", + "Seal_of_Kansas.svg", + "serialize-8888", + "svg", + "_", + "Seal_of_Kansas.svg" + ], + "env": { + "BUILDTYPE": "Debug", + "CHROME_HEADLESS": "1", + "SKIA_OUT": "[SLAVE_BUILD]/out" + }, + "name": "dm" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipes/swarm_test.py b/infra/bots/recipes/swarm_test.py index 1ceba3a1a2..67c9f2745d 100644 --- a/infra/bots/recipes/swarm_test.py +++ b/infra/bots/recipes/swarm_test.py @@ -431,8 +431,9 @@ def test_steps(api): ] if api.vars.is_trybot: properties.extend([ - 'issue', api.vars.issue, - 'patchset', api.vars.patchset, + 'issue', api.vars.issue, + 'patchset', api.vars.patchset, + 'patch_storage', api.vars.patch_storage, ]) args = [ @@ -860,3 +861,23 @@ def GenTests(api): ) + api.platform('win', 64) ) + + builder = 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug-Trybot' + gerrit_kwargs = { + 'patch_storage': 'gerrit', + 'repository': 'skia', + 'event.patchSet.ref': 'refs/changes/00/2100/2', + 'event.change.number': '2100', + } + yield ( + api.test('recipe_with_gerrit_patch') + + api.properties( + buildername=builder, + mastername='client.skia', + slavename='skiabot-linux-swarm-000', + buildnumber=5, + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]', + revision='abc123', + **gerrit_kwargs) + ) diff --git a/infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json b/infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json new file mode 100644 index 0000000000..246f91d326 --- /dev/null +++ b/infra/bots/recipes/swarm_trigger.expected/recipe_with_gerrit_patch.json @@ -0,0 +1,822 @@ +[ + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\n\nwith open(sys.argv[1]) as f:\n content = json.load(f)\n\nprint json.dumps(content, indent=2)\n", + "{\"buildername\": \"Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot\", \"buildnumber\": 5, \"event.change.number\": \"2100\", \"event.patchSet.ref\": \"refs/changes/00/2100/2\", \"mastername\": \"client.skia\", \"patch_storage\": \"gerrit\", \"path_config\": \"kitchen\", \"recipe\": \"swarm_trigger\", \"repository\": \"skia\", \"revision\": \"abc123\", \"slavename\": \"skiabot-linux-swarm-000\"}" + ], + "name": "print properties", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ content = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@print json.dumps(content, indent=2)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[SLAVE_BUILD]/skia", + "name": "git rev-parse", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-c", + "\"print 'abc123'\"" + ], + "name": "got_revision", + "~followup_annotations": [ + "@@@SET_BUILD_PROPERTY@got_revision@\"abc123\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "", + "[SLAVE_BUILD]/.gclient" + ], + "name": "write .gclient" + }, + { + "cmd": [ + "python", + "-u", + "import os\nfor r, _, files in os.walk(os.getcwd()):\n for fname in files:\n f = os.path.join(r, fname)\n if os.path.isfile(f):\n if os.access(f, os.X_OK):\n os.chmod(f, 0755)\n else:\n os.chmod(f, 0644)\n" + ], + "cwd": "[SLAVE_BUILD]/skia", + "name": "fix filemodes", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@for r, _, files in os.walk(os.getcwd()):@@@", + "@@@STEP_LOG_LINE@python.inline@ for fname in files:@@@", + "@@@STEP_LOG_LINE@python.inline@ f = os.path.join(r, fname)@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.isfile(f):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.access(f, os.X_OK):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.chmod(f, 0755)@@@", + "@@@STEP_LOG_LINE@python.inline@ else:@@@", + "@@@STEP_LOG_LINE@python.inline@ os.chmod(f, 0644)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[SLAVE_BUILD]/swarming.client", + "--url", + "https://chromium.googlesource.com/external/swarming.client.git" + ], + "name": "git setup (swarming_client)" + }, + { + "cmd": [ + "git", + "retry", + "fetch", + "origin", + "master" + ], + "cwd": "[SLAVE_BUILD]/swarming.client", + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "name": "git fetch (swarming_client)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[SLAVE_BUILD]/swarming.client", + "name": "git checkout (swarming_client)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[SLAVE_BUILD]/swarming.client", + "name": "read revision", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[SLAVE_BUILD]/swarming.client", + "name": "git clean (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[SLAVE_BUILD]/swarming.client", + "name": "submodule sync (swarming_client)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[SLAVE_BUILD]/swarming.client", + "name": "submodule update (swarming_client)" + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "--version" + ], + "name": "swarming.py --version", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_TEXT@0.8.6@@@" + ] + }, + { + "cmd": [ + "download_from_google_storage", + "--no_resume", + "--platform=linux*", + "--no_auth", + "--bucket", + "chromium-luci", + "-d", + "[SLAVE_BUILD]/skia/infra/bots/tools/luci-go/linux64" + ], + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "name": "download luci-go linux" + }, + { + "cmd": [ + "download_from_google_storage", + "--no_resume", + "--platform=darwin", + "--no_auth", + "--bucket", + "chromium-luci", + "-d", + "[SLAVE_BUILD]/skia/infra/bots/tools/luci-go/mac64" + ], + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "name": "download luci-go mac" + }, + { + "cmd": [ + "download_from_google_storage", + "--no_resume", + "--platform=win32", + "--no_auth", + "--bucket", + "chromium-luci", + "-d", + "[SLAVE_BUILD]/skia/infra/bots/tools/luci-go/win64" + ], + "env": { + "PATH": "RECIPE_PACKAGE_REPO[depot_tools]:%(PATH)s" + }, + "name": "download luci-go win" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", + "[SLAVE_BUILD]/luci-go" + ], + "env": { + "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" + }, + "name": "rmtree luci-go", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", + "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", + "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", + "[SLAVE_BUILD]/skia/infra/bots/tools/luci-go", + "[SLAVE_BUILD]/luci-go", + "0" + ], + "name": "Copy Go binary" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::gsutil]/resources/gsutil_wrapper.py", + "--", + "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py", + "----", + "help" + ], + "name": "gsutil help" + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" + ], + "name": "makedirs swarming tmp dir", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/compile_skia.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-compile_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Ubuntu\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/compile_skia.isolated.gen.json" + ], + "name": "Write compile_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/compile_skia.isolated.gen.json" + ], + "name": "isolate tests", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"compile_skia\": \"[dummy hash for compile_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"compile_skia\": \"[dummy hash for compile_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "compile_skia/Ubuntu/[dummy has/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Ubuntu", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot", + "--tag", + "buildnumber:5", + "--tag", + "data:[dummy hash for compile_skia]", + "--tag", + "master:client.skia", + "--tag", + "name:compile_skia", + "--tag", + "os:Ubuntu", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:compile_skia on Ubuntu", + "--idempotent", + "[dummy hash for compile_skia]", + "--", + "--workdir", + "../../..", + "swarm_compile", + "event.change.number=2100", + "repository=skia", + "buildername=Build-Ubuntu-GCC-x86_64-Debug-Trybot", + "mastername=client.skia.compile", + "patch_storage=gerrit", + "buildnumber=1", + "slavename=skiabot-dummy-compile-slave", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "event.patchSet.ref=refs/changes/00/2100/2", + "revision=abc123" + ], + "name": "[trigger] compile_skia on Ubuntu", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"compile_skia/Ubuntu/[dummy has/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"compile_skia/Ubuntu/[dummy has/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"compile_skia/Ubuntu/[dummy has/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\", \"tasks\": {\"compile_skia/Ubuntu/[dummy has/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json" + ], + "name": "compile_skia on Ubuntu", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[SLAVE_BUILD]/skia/infra/bots/assets/skp/VERSION", + "/path/to/tmp/" + ], + "name": "read skp VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[SLAVE_BUILD]/skia/infra/bots/assets/skimage/VERSION", + "/path/to/tmp/" + ], + "name": "read skimage VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "[SLAVE_BUILD]/skia/infra/bots/assets/svg/VERSION", + "/path/to/tmp/" + ], + "name": "read svg VERSION" + }, + { + "cmd": [ + "python", + "-u", + "\nimport sys, os\npath = sys.argv[1]\nmode = int(sys.argv[2])\nif not os.path.isdir(path):\n if os.path.exists(path):\n print \"%s exists but is not a dir\" % path\n sys.exit(1)\n os.makedirs(path, mode)\n", + "[SLAVE_BUILD]/swarming_temp_dir", + "511" + ], + "name": "makedirs swarming tmp dir (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import sys, os@@@", + "@@@STEP_LOG_LINE@python.inline@path = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@mode = int(sys.argv[2])@@@", + "@@@STEP_LOG_LINE@python.inline@if not os.path.isdir(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ if os.path.exists(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ print \"%s exists but is not a dir\" % path@@@", + "@@@STEP_LOG_LINE@python.inline@ sys.exit(1)@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(path, mode)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copy(sys.argv[1], sys.argv[2])\n", + "{\n \"args\": [\n \"--isolate\", \n \"[SLAVE_BUILD]/skia/infra/bots/test_skia.isolate\", \n \"--isolated\", \n \"[SLAVE_BUILD]/swarming_temp_dir/skia-task-test_skia.isolated\", \n \"--config-variable\", \n \"OS\", \n \"Ubuntu\", \n \"--blacklist\", \n \".git\", \n \"--blacklist\", \n \"out\", \n \"--blacklist\", \n \"*.pyc\", \n \"--blacklist\", \n \".recipe_deps\", \n \"--extra-variable\", \n \"WORKDIR\", \n \"[SLAVE_BUILD]\"\n ], \n \"dir\": \"[SLAVE_BUILD]\", \n \"version\": 1\n}", + "[SLAVE_BUILD]/swarming_temp_dir/test_skia.isolated.gen.json" + ], + "name": "Write test_skia.isolated.gen.json" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[build::isolate]/resources/isolate.py", + "[SLAVE_BUILD]/swarming.client", + "batcharchive", + "--dump-json", + "/path/to/tmp/json", + "--isolate-server", + "https://isolateserver.appspot.com", + "--verbose", + "[SLAVE_BUILD]/swarming_temp_dir/test_skia.isolated.gen.json" + ], + "name": "isolate tests (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_skia\": \"[dummy hash for test_skia]\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@swarm_hashes@{\"test_skia\": \"[dummy hash for test_skia]\"}@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport json\nimport sys\nwith open(sys.argv[1]) as f:\n isolated = json.load(f)\nif not isolated.get('includes'):\n isolated['includes'] = []\nfor h in sys.argv[2:]:\n isolated['includes'].append(h)\nwith open(sys.argv[1], 'w') as f:\n json.dump(isolated, f, sort_keys=True)\n", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-test_skia.isolated", + "abc123" + ], + "name": "add_isolated_input", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import json@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1]) as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated = json.load(f)@@@", + "@@@STEP_LOG_LINE@python.inline@if not isolated.get('includes'):@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'] = []@@@", + "@@@STEP_LOG_LINE@python.inline@for h in sys.argv[2:]:@@@", + "@@@STEP_LOG_LINE@python.inline@ isolated['includes'].append(h)@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ json.dump(isolated, f, sort_keys=True)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/isolateserver.py", + "archive", + "--isolate-server", + "https://isolateserver.appspot.com", + "[SLAVE_BUILD]/swarming_temp_dir/skia-task-test_skia.isolated" + ], + "name": "upload new .isolated file for test_skia", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "trigger", + "--swarming", + "https://chromium-swarm.appspot.com", + "--isolate-server", + "https://isolateserver.appspot.com", + "--priority", + "90", + "--shards", + "1", + "--task-name", + "test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5", + "--dump-json", + "/path/to/tmp/json", + "--expiration", + "72000", + "--io-timeout", + "2400", + "--hard-timeout", + "14400", + "--dimension", + "cpu", + "x86-64-avx2", + "--dimension", + "gpu", + "none", + "--dimension", + "os", + "Ubuntu", + "--dimension", + "pool", + "Skia", + "--tag", + "allow_milo:1", + "--tag", + "buildername:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot", + "--tag", + "buildnumber:5", + "--tag", + "data:def456", + "--tag", + "master:client.skia", + "--tag", + "name:test_skia", + "--tag", + "os:Ubuntu", + "--tag", + "revision:abc123", + "--tag", + "slavename:skiabot-linux-swarm-000", + "--tag", + "stepname:test_skia on Ubuntu", + "--cipd-package", + "skp:skia/bots/skp:version:0", + "--cipd-package", + "skimage:skia/bots/skimage:version:0", + "--cipd-package", + "svg:skia/bots/svg:version:0", + "def456", + "--", + "--workdir", + "../../..", + "swarm_test", + "event.change.number=2100", + "repository=skia", + "buildername=Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot", + "mastername=client.skia", + "patch_storage=gerrit", + "buildnumber=5", + "slavename=skiabot-linux-swarm-000", + "reason=Triggered by Skia swarm_trigger Recipe", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "event.patchSet.ref=refs/changes/00/2100/2", + "revision=abc123" + ], + "name": "[trigger] test_skia on Ubuntu", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"shard_index\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"10000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0@https://chromium-swarm.appspot.com/user/task/10000@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/10000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", + "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + ], + "env": { + "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" + }, + "name": "rmtree test_skia", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", + "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", + "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[SLAVE_BUILD]/swarming.client/swarming.py", + "collect", + "--swarming", + "https://chromium-swarm.appspot.com", + "--decorate", + "--print-status-updates", + "--json", + "{\"base_task_name\": \"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\", \"tasks\": {\"test_skia/Ubuntu/def456/Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot/5\": {\"shard_index\": 0, \"task_id\": \"10000\", \"view_url\": \"https://chromium-swarm.appspot.com/user/task/10000\"}}}", + "--task-summary-json", + "/path/to/tmp/json", + "--task-output-dir", + "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia" + ], + "name": "test_skia on Ubuntu", + "~followup_annotations": [ + "@@@STEP_TEXT@swarming pending 71s@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"shards\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"abandoned_ts\": null, @@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm30\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"completed_ts\": \"2014-09-25T01:42:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"created_ts\": \"2014-09-25T01:41:00.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"durations\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 5.7, @@@", + "@@@STEP_LOG_LINE@json.output@ 31.5@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_codes\": [@@@", + "@@@STEP_LOG_LINE@json.output@ 0, @@@", + "@@@STEP_LOG_LINE@json.output@ 0@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"id\": \"148aa78d7aa0000\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"internal_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated_out\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"view_url\": \"blah\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"modified_ts\": \"2014-09-25 01:42:00\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"heartbeat-canary-2014-09-25_01:41:55-os=Windows\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [@@@", + "@@@STEP_LOG_LINE@json.output@ \"Heart beat succeeded on win32.\\n\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"Foo\"@@@", + "@@@STEP_LOG_LINE@json.output@ ], @@@", + "@@@STEP_LOG_LINE@json.output@ \"started_ts\": \"2014-09-25T01:42:11.123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": 112, @@@", + "@@@STEP_LOG_LINE@json.output@ \"try_number\": 1, @@@", + "@@@STEP_LOG_LINE@json.output@ \"user\": \"unknown\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LINK@shard #0 isolated out@blah@@@", + "@@@STEP_LINK@view steps on Milo@https://luci-milo.appspot.com/swarming/task/148aa78d7aa0000@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport os, sys\nfrom common import chromium_utils # Error? See https://crbug.com/584783.\n\n\nif os.path.exists(sys.argv[1]):\n chromium_utils.RemoveDirectory(sys.argv[1])\n", + "[SLAVE_BUILD]/dm" + ], + "env": { + "PYTHONPATH": "[SLAVE_BUILD]/skia/infra/bots/.recipe_deps/build/scripts" + }, + "name": "rmtree dm", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os, sys@@@", + "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@", + "@@@STEP_LOG_LINE@python.inline@ chromium_utils.RemoveDirectory(sys.argv[1])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "\nimport shutil\nimport sys\nshutil.copytree(sys.argv[1], sys.argv[2], symlinks=bool(sys.argv[3]))\n", + "[SLAVE_BUILD]/swarming_temp_dir/outputs/test_skia/0/dm", + "[SLAVE_BUILD]/dm", + "0" + ], + "name": "dm_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::core]/resources/upload_dm_results.py", + "[SLAVE_BUILD]/dm", + "abc123", + "Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot", + "5", + "2100", + "[SLAVE_BUILD]/skia/common/py/utils" + ], + "cwd": "[SLAVE_BUILD]/skia", + "env": { + "AWS_CREDENTIAL_FILE": "[HOME]/chromium-skia-gm.boto", + "BOTO_CONFIG": "[HOME]/chromium-skia-gm.boto" + }, + "name": "Upload DM Results" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipes/swarm_trigger.py b/infra/bots/recipes/swarm_trigger.py index b16405dd06..f49cd299f8 100644 --- a/infra/bots/recipes/swarm_trigger.py +++ b/infra/bots/recipes/swarm_trigger.py @@ -193,9 +193,15 @@ def trigger_task(api, task_name, builder, master, slave, buildnumber, 'swarm_out_dir': '${ISOLATED_OUTDIR}', } if builder_cfg['is_trybot']: - properties['issue'] = str(api.properties['issue']) - properties['patchset'] = str(api.properties['patchset']) - properties['rietveld'] = api.properties['rietveld'] + if api.properties.get('patch_storage') == 'gerrit': + properties['patch_storage'] = api.properties['patch_storage'] + properties['repository'] = api.properties['repository'] + properties['event.patchSet.ref'] = api.properties['event.patchSet.ref'] + properties['event.change.number'] = api.properties['event.change.number'] + else: + properties['issue'] = str(api.properties['issue']) + properties['patchset'] = str(api.properties['patchset']) + properties['rietveld'] = api.properties['rietveld'] extra_args = [ '--workdir', '../../..', @@ -418,6 +424,13 @@ def gsutil_env(api, boto_file): 'BOTO_CONFIG': boto_path} +def get_issue_num(api): + if api.properties.get('patch_storage') == 'gerrit': + return str(api.properties['event.change.number']) + else: + return str(api.properties['issue']) + + def perf_steps_trigger(api, builder_cfg, got_revision, infrabots_dir, extra_hashes, cipd_packages): """Trigger perf tests via Swarming.""" @@ -468,7 +481,7 @@ def perf_steps_collect(api, task, got_revision, is_trybot): upload_args = [api.properties['buildername'], api.properties['buildnumber'], perf_data_dir, got_revision, gsutil_path] if is_trybot: - upload_args.append(api.properties['issue']) + upload_args.append(get_issue_num(api)) api.python( 'Upload perf results', script=api.core.resource('upload_bench_results.py'), @@ -520,7 +533,7 @@ def test_steps_collect(api, task, got_revision, is_trybot, builder_cfg): got_revision, api.properties['buildername'], api.properties['buildnumber'], - api.properties['issue'] if is_trybot else '', + get_issue_num(api) if is_trybot else '', api.path['slave_build'].join('skia', 'common', 'py', 'utils'), ], cwd=api.path['checkout'], @@ -546,8 +559,7 @@ def upload_coverage_results(api, task, got_revision, is_trybot): api.properties['buildername'], str(api.properties['buildnumber']))) if is_trybot: - gs_json_path = '/'.join(('trybot', gs_json_path, - str(api.properties['issue']))) + gs_json_path = '/'.join(('trybot', gs_json_path, get_issue_num(api))) api.gsutil.upload( name='upload raw coverage data', source=cov_file, @@ -572,7 +584,7 @@ def upload_coverage_results(api, task, got_revision, is_trybot): upload_args = [api.properties['buildername'], api.properties['buildnumber'], results_dir, got_revision, gsutil_path] if is_trybot: - upload_args.append(api.properties['issue']) + upload_args.append(get_issue_num(api)) api.python( 'upload nanobench coverage results', script=api.core.resource('upload_bench_results.py'), @@ -773,3 +785,24 @@ def GenTests(api): for slavename, builders_by_slave in slaves.iteritems(): for builder in builders_by_slave: yield test_for_bot(api, builder, mastername, slavename) + + gerrit_kwargs = { + 'patch_storage': 'gerrit', + 'repository': 'skia', + 'event.patchSet.ref': 'refs/changes/00/2100/2', + 'event.change.number': '2100', + } + yield ( + api.test('recipe_with_gerrit_patch') + + api.properties( + buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-Trybot', + mastername='client.skia', + slavename='skiabot-linux-swarm-000', + buildnumber=5, + path_config='kitchen', + revision='abc123', + **gerrit_kwargs) + + api.step_data( + 'upload new .isolated file for test_skia', + stdout=api.raw_io.output('def456 XYZ.isolated')) + ) |